source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.metadata.repository.jcr;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import com.google.common.collect.ImmutableSet;
  23 import org.apache.jackrabbit.JcrConstants;
  24 import org.apache.jackrabbit.oak.Oak;
  25 import org.apache.jackrabbit.oak.api.Type;
  26 import org.apache.jackrabbit.oak.jcr.Jcr;
  27 import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
  28 import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
  29 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
  30 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
  31 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
  32 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
  33 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
  34 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver;
  35 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory;
  36 import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
  37 import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
  38 import org.apache.jackrabbit.oak.segment.file.FileStore;
  39 import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
  40 import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
  41 import org.apache.jackrabbit.oak.spi.commit.Observer;
  42 import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
  43 import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
  44 import org.apache.jackrabbit.oak.spi.mount.Mounts;
  45 import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
  46 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
  47 import org.apache.jackrabbit.oak.spi.state.NodeStore;
  48 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51
  52 import javax.annotation.Nonnull;
  53 import javax.jcr.Repository;
  54 import java.io.File;
  55 import java.io.IOException;
  56 import java.nio.file.Files;
  57 import java.nio.file.Path;
  58 import java.nio.file.Paths;
  59 import java.util.concurrent.ExecutorService;
  60 import java.util.concurrent.LinkedBlockingQueue;
  61 import java.util.concurrent.ThreadFactory;
  62 import java.util.concurrent.ThreadPoolExecutor;
  63 import java.util.concurrent.TimeUnit;
  64 import java.util.concurrent.atomic.AtomicInteger;
  65
  66 import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
  67
  68 /**
  69  * Created by martin on 14.06.17.
  70  *
  71  * @author Martin Stockhammer
  72  * @since 3.0.0
  73  */
  74 public class RepositoryFactory
  75 {
  76
  77     Logger log = LoggerFactory.getLogger( RepositoryFactory.class );
  78
  79     public static final String SEGMENT_FILE_TYPE = "oak-segment-tar";
  80     public static final String IN_MEMORY_TYPE = "oak-memory";
  81
  82     String storeType = SEGMENT_FILE_TYPE;
  83
  84     Path repositoryPath = Paths.get( "repository" );
  85
  86     public Repository createRepository( ) throws IOException, InvalidFileStoreVersionException
  87     {
  88         NodeStore nodeStore;
  89         if ( SEGMENT_FILE_TYPE.equals( storeType ) )
  90         {
  91             FileStore fs = FileStoreBuilder.fileStoreBuilder( repositoryPath.toFile( ) ).build( );
  92             nodeStore = SegmentNodeStoreBuilders.builder( fs ).build( );
  93         } else if (IN_MEMORY_TYPE.equals(storeType)) {
  94             nodeStore = null;
  95         } else {
  96             throw new IllegalArgumentException( "Store type "+storeType+" not recognized" );
  97         }
  98
  99         Oak oak = nodeStore==null ? new Oak() : new Oak(nodeStore);
 100         oak.with( new RepositoryInitializer( )
 101         {
 102             @Override
 103             public void initialize( @Nonnull NodeBuilder root )
 104             {
 105                 log.info("Creating index ");
 106
 107                 NodeBuilder lucene = IndexUtils.getOrCreateOakIndex( root ).child("lucene");
 108                 lucene.setProperty( JcrConstants.JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", Type.NAME);
 109
 110                 lucene.setProperty("compatVersion", 2);
 111                 lucene.setProperty("type", "lucene");
 112                 // lucene.setProperty("async", "async");
 113                 lucene.setProperty(INCLUDE_PROPERTY_TYPES,
 114                     ImmutableSet.of("String"), Type.STRINGS);
 115                 // lucene.setProperty("refresh",true);
 116                 lucene.setProperty("async",ImmutableSet.of("async", "sync"), Type.STRINGS);
 117                 NodeBuilder rules = lucene.child("indexRules").
 118                     setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
 119                 rules.setProperty(":childOrder",ImmutableSet.of("archiva:projectVersion","archiva:artifact",
 120                     "archiva:facet","archiva:namespace", "archiva:project"), Type.STRINGS);
 121                 NodeBuilder allProps = rules.child("archiva:projectVersion")
 122                     .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE,
 123                         "nt:unstructured", Type.NAME)
 124                     .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS )
 125                     .setProperty("indexNodeName",true)
 126                     .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
 127                 allProps.setProperty("name", ".*");
 128                 allProps.setProperty("isRegexp", true);
 129                 allProps.setProperty("nodeScopeIndex", true);
 130                 allProps.setProperty("index",true);
 131                 allProps.setProperty("analyzed",true);
 132                 // allProps.setProperty("propertyIndex",true);
 133                 allProps = rules.child("archiva:artifact")
 134                     .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE,
 135                         "nt:unstructured", Type.NAME)
 136                     .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS )
 137                     .setProperty("indexNodeName",true)
 138                     .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
 139                 allProps.setProperty("name", ".*");
 140                 allProps.setProperty("isRegexp", true);
 141                 allProps.setProperty("nodeScopeIndex", true);
 142                 allProps.setProperty("index",true);
 143                 allProps.setProperty("analyzed",true);
 144                 allProps = rules.child("archiva:facet")
 145                     .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE,
 146                         "nt:unstructured", Type.NAME)
 147                     .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS )
 148                     .setProperty("indexNodeName",true)
 149                     .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
 150                 allProps.setProperty("name", ".*");
 151                 allProps.setProperty("isRegexp", true);
 152                 allProps.setProperty("nodeScopeIndex", true);
 153                 allProps.setProperty("index",true);
 154                 allProps.setProperty("analyzed",true);
 155                 allProps = rules.child("archiva:namespace")
 156                     .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE,
 157                         "nt:unstructured", Type.NAME)
 158                     .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS )
 159                     .setProperty("indexNodeName",true)
 160                     .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
 161                 allProps.setProperty("name", ".*");
 162                 allProps.setProperty("isRegexp", true);
 163                 allProps.setProperty("nodeScopeIndex", true);
 164                 allProps.setProperty("index",true);
 165                 allProps.setProperty("analyzed",true);
 166                 allProps = rules.child("archiva:project")
 167                     .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE,
 168                         "nt:unstructured", Type.NAME)
 169                     .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS )
 170                     .setProperty("indexNodeName",true)
 171                     .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
 172                 allProps.setProperty("name", ".*");
 173                 allProps.setProperty("isRegexp", true);
 174                 allProps.setProperty("nodeScopeIndex", true);
 175                 allProps.setProperty("index",true);
 176                 allProps.setProperty("analyzed",true);
 177
 178                 log.info("Index: "+lucene+" myIndex "+lucene.getChildNode( "myIndex" ));
 179                 log.info("myIndex "+lucene.getChildNode( "myIndex" ).getProperties());
 180                 // IndexUtils.createIndexDefinition(  )
 181
 182             }
 183         } );
 184
 185         ExecutorService executorService = createExecutor();
 186         StatisticsProvider statsProvider = StatisticsProvider.NOOP;
 187         int queueSize = Integer.getInteger("queueSize", 10000);
 188         File indexDir = Files.createTempDirectory( "archiva_index" ).toFile();
 189         log.info("Queue Index "+indexDir.toString());
 190         IndexCopier indexCopier = new IndexCopier( executorService, indexDir, true );
 191         NRTIndexFactory nrtIndexFactory = new NRTIndexFactory( indexCopier, statsProvider);
 192         MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider( );
 193         IndexTracker tracker = new IndexTracker(new DefaultIndexReaderFactory( mountInfoProvider, indexCopier ), nrtIndexFactory);
 194         DocumentQueue queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider);
 195         LocalIndexObserver localIndexObserver = new LocalIndexObserver( queue, statsProvider);
 196         LuceneIndexProvider provider = new LuceneIndexProvider(tracker);
 197
 198         //        ExternalObserverBuilder builder = new ExternalObserverBuilder(queue, tracker, statsProvider,
 199 //            executorService, queueSize);
 200 //        Observer observer = builder.build();
 201 //        builder.getBackgroundObserver();
 202
 203         LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(null,
 204         tracker, new ExtractedTextCache(0, 0), null, mountInfoProvider);
 205         editorProvider.setIndexingQueue(queue);
 206
 207
 208         log.info("Oak: "+oak+" with nodeStore "+nodeStore);
 209         Jcr jcr = new Jcr(oak).with( editorProvider ).with( (Observer) provider )
 210             .with(localIndexObserver)
 211             // .with(observer)
 212             .with( ( QueryIndexProvider) provider )
 213             .withAsyncIndexing("async",5  );
 214         Repository r = jcr.createRepository();
 215         try
 216         {
 217             Thread.currentThread().sleep(1000);
 218         }
 219         catch ( InterruptedException e )
 220         {
 221             e.printStackTrace( );
 222         }
 223         return r;
 224
 225
 226     }
 227
 228     public String getStoreType( )
 229     {
 230         return storeType;
 231     }
 232
 233     public void setStoreType( String storeType )
 234     {
 235         this.storeType = storeType;
 236     }
 237
 238     public Path getRepositoryPath( )
 239     {
 240         return repositoryPath;
 241     }
 242
 243     public void setRepositoryPath( Path repositoryPath )
 244     {
 245         this.repositoryPath = repositoryPath;
 246     }
 247
 248     public void setRepositoryPath( String repositoryPath )
 249     {
 250         this.repositoryPath = Paths.get( repositoryPath );
 251         if ( !Files.exists( this.repositoryPath ) )
 252         {
 253             try
 254             {
 255                 Files.createDirectories( this.repositoryPath );
 256             }
 257             catch ( IOException e )
 258             {
 259                 e.printStackTrace( );
 260             }
 261         }
 262     }
 263
 264     private ExecutorService createExecutor() {
 265         ThreadPoolExecutor executor = new ThreadPoolExecutor(0, 5, 60L, TimeUnit.SECONDS,
 266             new LinkedBlockingQueue<Runnable>(), new ThreadFactory() {
 267             private final AtomicInteger counter = new AtomicInteger();
 268             private final Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
 269                 @Override
 270                 public void uncaughtException(Thread t, Throwable e) {
 271                     log.warn("Error occurred in asynchronous processing ", e);
 272                 }
 273             };
 274             @Override
 275             public Thread newThread(@Nonnull Runnable r) {
 276                 Thread thread = new Thread(r, createName());
 277                 thread.setDaemon(true);
 278                 thread.setPriority(Thread.MIN_PRIORITY);
 279                 thread.setUncaughtExceptionHandler(handler);
 280                 return thread;
 281             }
 282
 283             private String createName() {
 284                 return "oak-lucene-" + counter.getAndIncrement();
 285             }
 286         });
 287         executor.setKeepAliveTime(1, TimeUnit.MINUTES);
 288         executor.allowCoreThreadTimeOut(true);
 289         return executor;
 290     }
 291
 292 }