source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.metadata.repository.jcr;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import com.google.common.collect.ImmutableList;
  23 import com.google.common.collect.ImmutableSet;
  24 import org.apache.commons.lang.time.StopWatch;
  25 import org.apache.jackrabbit.oak.api.Type;
  26 import org.apache.jackrabbit.oak.jcr.Jcr;
  27 import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
  28 import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoServiceImpl;
  29 import org.apache.jackrabbit.oak.plugins.index.IndexInfoProvider;
  30 import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
  31 import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl;
  32 import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
  33 import org.apache.jackrabbit.oak.plugins.index.aggregate.SimpleNodeAggregator;
  34 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexAugmentorFactory;
  35 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
  36 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
  37 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
  38 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexInfoProvider;
  39 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
  40 import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory;
  41 import org.apache.jackrabbit.oak.plugins.index.lucene.directory.BufferedOakDirectory;
  42 import org.apache.jackrabbit.oak.plugins.index.lucene.directory.LuceneIndexImporter;
  43 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
  44 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.ExternalObserverBuilder;
  45 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver;
  46 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory;
  47 import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner;
  48 import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
  49 import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory;
  50 import org.apache.jackrabbit.oak.plugins.index.lucene.score.impl.ScorerProviderFactoryImpl;
  51 import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
  52 import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
  53 import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
  54 import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
  55 import org.apache.jackrabbit.oak.plugins.name.Namespaces;
  56 import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
  57 import org.apache.jackrabbit.oak.segment.file.FileStore;
  58 import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
  59 import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
  60 import org.apache.jackrabbit.oak.spi.blob.FileBlobStore;
  61 import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
  62 import org.apache.jackrabbit.oak.spi.commit.BackgroundObserver;
  63 import org.apache.jackrabbit.oak.spi.commit.Observer;
  64 import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
  65 import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
  66 import org.apache.jackrabbit.oak.spi.mount.Mounts;
  67 import org.apache.jackrabbit.oak.spi.namespace.NamespaceConstants;
  68 import org.apache.jackrabbit.oak.spi.query.QueryIndex;
  69 import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
  70 import org.apache.jackrabbit.oak.spi.state.Clusterable;
  71 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
  72 import org.apache.jackrabbit.oak.spi.state.NodeStore;
  73 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
  74 import org.jetbrains.annotations.NotNull;
  75 import org.slf4j.Logger;
  76 import org.slf4j.LoggerFactory;
  77
  78 import javax.annotation.Nonnull;
  79 import javax.jcr.Repository;
  80 import java.io.IOException;
  81 import java.nio.file.Files;
  82 import java.nio.file.Path;
  83 import java.nio.file.Paths;
  84 import java.util.concurrent.ExecutorService;
  85 import java.util.concurrent.LinkedBlockingQueue;
  86 import java.util.concurrent.ThreadFactory;
  87 import java.util.concurrent.ThreadPoolExecutor;
  88 import java.util.concurrent.TimeUnit;
  89 import java.util.concurrent.atomic.AtomicInteger;
  90
  91 import static com.google.common.base.Preconditions.checkNotNull;
  92 import static org.apache.archiva.metadata.repository.jcr.OakRepositoryFactory.StoreType.IN_MEMORY_TYPE;
  93 import static org.apache.archiva.metadata.repository.jcr.OakRepositoryFactory.StoreType.SEGMENT_FILE_TYPE;
  94 import static org.apache.commons.io.FileUtils.ONE_MB;
  95 import static org.apache.jackrabbit.JcrConstants.*;
  96 import static org.apache.jackrabbit.oak.api.Type.NAME;
  97 import static org.apache.archiva.metadata.repository.jcr.JcrConstants.*;
  98 import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.registerMBean;
  99 import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.scheduleWithFixedDelay;
 100
 101 /**
 102  * Created by martin on 14.06.17.
 103  *
 104  * @author Martin Stockhammer
 105  * @since 3.0.0
 106  */
 107 public class OakRepositoryFactory
 108 {
 109
 110     private Logger log = LoggerFactory.getLogger( OakRepositoryFactory.class );
 111
 112     private FileStore fileStore;
 113
 114     private NodeStore nodeStore;
 115
 116     private IndexTracker tracker;
 117
 118     private DocumentQueue documentQueue;
 119
 120     private NRTIndexFactory nrtIndexFactory;
 121
 122     private IndexCopier indexCopier;
 123
 124     private ExecutorService executorService;
 125     private ExtractedTextCache extractedTextCache;
 126
 127     private boolean hybridIndex = true;
 128     private boolean prefetchEnabled = true;
 129     private boolean enableAsyncIndexOpen = true;
 130     int queueSize = 10000;
 131     int cleanerInterval = 10*60;
 132     boolean enableCopyOnWrite = true;
 133     boolean enableCopyOnRead = true;
 134     int cacheSizeInMB = 20;
 135     int cacheExpiryInSecs = 300;
 136     int threadPoolSize = 5;
 137
 138     private StatisticsProvider statisticsProvider;
 139
 140     private MountInfoProvider mountInfoProvider =  Mounts.defaultMountInfoProvider();
 141
 142     private AsyncIndexInfoService asyncIndexInfoService = null;
 143
 144     private LuceneIndexProvider indexProvider;
 145
 146     private ScorerProviderFactory scorerFactory = new ScorerProviderFactoryImpl( );
 147     private IndexAugmentorFactory augmentorFactory = new IndexAugmentorFactory( );
 148
 149     private ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollector activeDeletedBlobCollector = ActiveDeletedBlobCollectorFactory.NOOP;
 150
 151     private QueryIndex.NodeAggregator nodeAggregator = new SimpleNodeAggregator( );
 152
 153     private BackgroundObserver backgroundObserver;
 154
 155     private BackgroundObserver externalIndexObserver;
 156
 157     private GarbageCollectableBlobStore blobStore;
 158
 159     private PropertyIndexCleaner cleaner;
 160
 161     private IndexPathService indexPathService;
 162
 163     private LuceneIndexEditorProvider editorProvider;
 164
 165     private Path indexDir;
 166
 167     public enum StoreType
 168     {
 169         SEGMENT_FILE_TYPE,
 170         IN_MEMORY_TYPE;
 171     }
 172
 173     private StoreType storeType = SEGMENT_FILE_TYPE;
 174
 175     private Path repositoryPath = Paths.get( "repository" );
 176
 177     public OakRepositoryFactory() {
 178         final OakRepositoryFactory repositoryFactory = this;
 179         Runtime.getRuntime().addShutdownHook( new Thread( ( ) -> {
 180             if (repositoryFactory!=null)
 181             {
 182                 repositoryFactory.close( );
 183             }
 184         } ) );
 185     }
 186
 187     private void initializeExtractedTextCache( StatisticsProvider statisticsProvider) {
 188         boolean alwaysUsePreExtractedCache = false;
 189
 190         extractedTextCache = new ExtractedTextCache(
 191             cacheSizeInMB * ONE_MB,
 192             cacheExpiryInSecs,
 193             alwaysUsePreExtractedCache,
 194             indexDir.toFile(), statisticsProvider);
 195     }
 196
 197     private IndexTracker createTracker() throws IOException {
 198         IndexTracker tracker;
 199         if (enableCopyOnRead){
 200             initializeIndexCopier();
 201             log.info("Enabling CopyOnRead support. Index files would be copied under {}", indexDir.toAbsolutePath());
 202             if (hybridIndex) {
 203                 nrtIndexFactory = new NRTIndexFactory(indexCopier, statisticsProvider);
 204             }
 205             tracker = new IndexTracker(new DefaultIndexReaderFactory(mountInfoProvider, indexCopier), nrtIndexFactory);
 206         } else {
 207             tracker = new IndexTracker(new DefaultIndexReaderFactory(mountInfoProvider, null));
 208         }
 209
 210         tracker.setAsyncIndexInfoService(asyncIndexInfoService);
 211         tracker.refresh();
 212         return tracker;
 213     }
 214
 215     private void initializeIndexCopier() throws IOException {
 216         if(indexCopier != null){
 217             return;
 218         }
 219
 220         if (prefetchEnabled){
 221             log.info("Prefetching of index files enabled. Index would be opened after copying all new files locally");
 222         }
 223
 224         indexCopier = new IndexCopier(getExecutorService(), indexDir.toFile(), prefetchEnabled);
 225
 226     }
 227
 228     ExecutorService getExecutorService(){
 229         if (executorService == null){
 230             executorService = createExecutor();
 231         }
 232         return executorService;
 233     }
 234
 235     private ExecutorService createExecutor() {
 236         ThreadPoolExecutor executor = new ThreadPoolExecutor(threadPoolSize, threadPoolSize, 60L, TimeUnit.SECONDS,
 237             new LinkedBlockingQueue<Runnable>(), new ThreadFactory() {
 238             private final AtomicInteger counter = new AtomicInteger();
 239             private final Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
 240                 @Override
 241                 public void uncaughtException(Thread t, Throwable e) {
 242                     log.warn("Error occurred in asynchronous processing ", e);
 243                 }
 244             };
 245             @Override
 246             public Thread newThread(@NotNull Runnable r) {
 247                 Thread thread = new Thread(r, createName());
 248                 thread.setDaemon(true);
 249                 thread.setPriority(Thread.MIN_PRIORITY);
 250                 thread.setUncaughtExceptionHandler(handler);
 251                 return thread;
 252             }
 253
 254             private String createName() {
 255                 return "oak-lucene-" + counter.getAndIncrement();
 256             }
 257         });
 258         executor.setKeepAliveTime(1, TimeUnit.MINUTES);
 259         executor.allowCoreThreadTimeOut(true);
 260         return executor;
 261     }
 262
 263     private void initialize(){
 264         if(indexProvider == null){
 265             return;
 266         }
 267
 268         if(nodeAggregator != null){
 269             log.debug("Using NodeAggregator {}", nodeAggregator.getClass());
 270         }
 271
 272         indexProvider.setAggregator(nodeAggregator);
 273     }
 274
 275     private void registerObserver() {
 276         Observer observer = indexProvider;
 277         if (enableAsyncIndexOpen) {
 278             backgroundObserver = new BackgroundObserver(indexProvider, getExecutorService(), 5);
 279             log.info("Registering the LuceneIndexProvider as a BackgroundObserver");
 280         }
 281     }
 282
 283     private void registerLocalIndexObserver(IndexTracker tracker) {
 284         if (!hybridIndex){
 285             log.info("Hybrid indexing feature disabled");
 286             return;
 287         }
 288         documentQueue = new DocumentQueue( queueSize, tracker, getExecutorService(), statisticsProvider);
 289         LocalIndexObserver localIndexObserver = new LocalIndexObserver(documentQueue, statisticsProvider);
 290
 291         int observerQueueSize = 1000;
 292         int builderMaxSize = 5000;
 293         // regs.add(bundleContext.registerService(JournalPropertyService.class.getName(),
 294         //    new LuceneJournalPropertyService(builderMaxSize), null));
 295         ExternalObserverBuilder builder = new ExternalObserverBuilder(documentQueue, tracker, statisticsProvider,
 296             getExecutorService(), observerQueueSize);
 297         log.info("Configured JournalPropertyBuilder with max size {} and backed by BackgroundObserver " +
 298             "with queue size {}", builderMaxSize, observerQueueSize);
 299
 300         Observer observer = builder.build();
 301         externalIndexObserver = builder.getBackgroundObserver();
 302         log.info("Hybrid indexing enabled for configured indexes with queue size of {}", queueSize );
 303     }
 304
 305     private IndexInfoProvider registerIndexInfoProvider() {
 306         return new LuceneIndexInfoProvider(nodeStore, asyncIndexInfoService, getIndexCheckDir().toFile());
 307     }
 308
 309     private Path getIndexCheckDir() {
 310         return checkNotNull(indexDir).resolve("indexCheckDir");
 311     }
 312
 313     private LuceneIndexImporter registerIndexImporterProvider() {
 314         return new LuceneIndexImporter(blobStore);
 315     }
 316
 317     private void registerPropertyIndexCleaner( ) {
 318
 319         if (cleanerInterval <= 0) {
 320             log.info("Property index cleaner would not be registered");
 321             return;
 322         }
 323
 324         cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService, statisticsProvider);
 325
 326         //Proxy check for DocumentNodeStore
 327         if (nodeStore instanceof Clusterable ) {
 328             cleaner.setRecursiveDelete(true);
 329             log.info("PropertyIndexCleaner configured to perform recursive delete");
 330         }
 331         log.info("Property index cleaner configured to run every [{}] seconds", cleanerInterval);
 332     }
 333
 334     private void registerIndexEditor( IndexTracker tracker) throws IOException {
 335         boolean enableCopyOnWrite = true;
 336         if (enableCopyOnWrite){
 337             initializeIndexCopier();
 338             editorProvider = new LuceneIndexEditorProvider(indexCopier, tracker, extractedTextCache,
 339                 augmentorFactory,  mountInfoProvider, activeDeletedBlobCollector, null, statisticsProvider);
 340             log.info("Enabling CopyOnWrite support. Index files would be copied under {}", indexDir.toAbsolutePath());
 341         } else {
 342             editorProvider = new LuceneIndexEditorProvider(null, tracker, extractedTextCache, augmentorFactory,
 343                 mountInfoProvider, activeDeletedBlobCollector, null, statisticsProvider);
 344         }
 345         editorProvider.setBlobStore(blobStore);
 346
 347         if (hybridIndex){
 348             editorProvider.setIndexingQueue(checkNotNull(documentQueue));
 349         }
 350
 351
 352     }
 353
 354     public Repository createRepository()
 355         throws IOException, InvalidFileStoreVersionException
 356     {
 357
 358         indexDir = repositoryPath.resolve( ".index-lucene" );
 359         if (!Files.exists( indexDir )) {
 360             Files.createDirectories( indexDir );
 361         }
 362         blobStore = new FileBlobStore( indexDir.resolve( "blobs" ).toAbsolutePath().toString() );
 363
 364         statisticsProvider = StatisticsProvider.NOOP;
 365
 366         if ( SEGMENT_FILE_TYPE == storeType )
 367         {
 368             fileStore = FileStoreBuilder.fileStoreBuilder( repositoryPath.toFile() )
 369                 .withStatisticsProvider( statisticsProvider )
 370                 .build();
 371             nodeStore = SegmentNodeStoreBuilders.builder( fileStore ) //
 372                 .withStatisticsProvider( statisticsProvider ) //
 373                 .build();
 374         }
 375         else if ( IN_MEMORY_TYPE == storeType )
 376         {
 377             nodeStore = new MemoryNodeStore( );
 378         }
 379         else
 380         {
 381             throw new IllegalArgumentException( "Store type " + storeType + " not recognized" );
 382         }
 383
 384         asyncIndexInfoService = new AsyncIndexInfoServiceImpl( nodeStore );
 385
 386         indexPathService = new IndexPathServiceImpl( nodeStore, mountInfoProvider );
 387
 388         BufferedOakDirectory.setEnableWritingSingleBlobIndexFile( true );
 389
 390         initializeExtractedTextCache( statisticsProvider );
 391
 392         tracker = createTracker();
 393
 394         indexProvider = new LuceneIndexProvider(tracker, scorerFactory, augmentorFactory);
 395
 396         initialize();
 397         registerObserver();
 398         registerLocalIndexObserver(tracker);
 399         registerIndexInfoProvider();
 400         registerIndexImporterProvider();
 401         registerPropertyIndexCleaner();
 402
 403         registerIndexEditor(tracker);
 404
 405
 406
 407         RepositoryInitializer repoInitializer = new RepositoryInitializer( )
 408         {
 409             private IndexDefinitionBuilder.PropertyRule initRegexAll( IndexDefinitionBuilder.IndexRule rule ) {
 410                 return rule
 411                     .indexNodeName( )
 412                     .property(JCR_LASTMODIFIED ).propertyIndex().type( "Date" ).ordered()
 413                     .property(JCR_PRIMARYTYPE).propertyIndex()
 414                     .property(JCR_MIXINTYPES).propertyIndex()
 415                     .property(JCR_PATH).propertyIndex().ordered()
 416                     .property( FulltextIndexConstants.REGEX_ALL_PROPS, true )
 417                     .propertyIndex().analyzed( ).nodeScopeIndex();
 418             }
 419
 420             private IndexDefinitionBuilder.PropertyRule initBaseRule( IndexDefinitionBuilder.IndexRule rule ) {
 421                 return rule
 422                     .sync()
 423                     .indexNodeName( )
 424                     .property(JCR_CREATED).propertyIndex().type("Date").ordered()
 425                     .property(JCR_LASTMODIFIED ).propertyIndex().type( "Date" ).ordered()
 426                     .property(JCR_PRIMARYTYPE).propertyIndex()
 427                     .property(JCR_MIXINTYPES).propertyIndex()
 428                     .property(JCR_PATH).propertyIndex().ordered()
 429                     .property( "id" ).propertyIndex().analyzed( );
 430             }
 431
 432             @Override
 433             public void initialize( @Nonnull NodeBuilder root )
 434             {
 435                 NodeBuilder namespaces;
 436                 if ( !root.hasChildNode( NamespaceConstants.REP_NAMESPACES ) )
 437                 {
 438                     namespaces = Namespaces.createStandardMappings( root );
 439                     Namespaces.buildIndexNode( namespaces ); // index node for faster lookup
 440                 }
 441                 else
 442                 {
 443                     namespaces = root.getChildNode( NamespaceConstants.REP_NAMESPACES );
 444                 }
 445                 Namespaces.addCustomMapping( namespaces, "http://archiva.apache.org/jcr/", "archiva" );
 446
 447                 log.info( "Creating index " );
 448
 449                 NodeBuilder oakIdx = IndexUtils.getOrCreateOakIndex( root );
 450                 if (!oakIdx.hasChildNode( "repo-lucene" ))
 451                 {
 452                     NodeBuilder lucene = oakIdx.child( "repo-lucene" );
 453                     lucene.setProperty( JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", NAME );
 454
 455                     lucene.setProperty( "compatVersion", 2 );
 456                     lucene.setProperty( "type", "lucene" );
 457                     // lucene.setProperty("async", "async");
 458                     // lucene.setProperty( INCLUDE_PROPERTY_TYPES, ImmutableSet.of(  ), Type.STRINGS );
 459                     // lucene.setProperty("refresh",true);
 460                     NodeBuilder rules = lucene.child( "indexRules" ).
 461                         setProperty( JCR_PRIMARYTYPE, NT_UNSTRUCTURED, NAME );
 462                     rules.setProperty( ":childOrder", ImmutableSet.of(
 463                         REPOSITORY_NODE_TYPE,
 464                         NAMESPACE_NODE_TYPE, //
 465                         PROJECT_NODE_TYPE,
 466                         PROJECT_VERSION_NODE_TYPE, //
 467                         ARTIFACT_NODE_TYPE, //
 468                         FACET_NODE_TYPE //
 469                     ), Type.STRINGS );
 470                     IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder( lucene );
 471                     idxBuilder.async( "async", "nrt" ).includedPaths( "/repositories" ).evaluatePathRestrictions();
 472
 473                     initBaseRule(idxBuilder.indexRule( REPOSITORY_NODE_TYPE ));
 474                     initBaseRule(idxBuilder.indexRule( NAMESPACE_NODE_TYPE ))
 475                         .property( "namespace" ).propertyIndex().analyzed();
 476                     initBaseRule(idxBuilder.indexRule( PROJECT_NODE_TYPE ))
 477                         .property( "name" ).propertyIndex().analyzed().notNullCheckEnabled().nullCheckEnabled();
 478                     initBaseRule( idxBuilder.indexRule( PROJECT_VERSION_NODE_TYPE ) )
 479                         .property("name").propertyIndex().analyzed().notNullCheckEnabled().nullCheckEnabled()
 480                         .property("description").propertyIndex().analyzed().notNullCheckEnabled().nullCheckEnabled()
 481                         .property("url").propertyIndex().analyzed( ).notNullCheckEnabled().nullCheckEnabled()
 482                         .property("incomplete").type("Boolean").propertyIndex()
 483                         .property("mailinglist/name").propertyIndex().analyzed()
 484                         .property("license/license.name").propertyIndex().analyzed();
 485                     initBaseRule(idxBuilder.indexRule( ARTIFACT_NODE_TYPE ))
 486                         .property( "whenGathered" ).type("Date").propertyIndex().analyzed().ordered()
 487                         .property("size").type("Long").propertyIndex().analyzed().ordered()
 488                         .property("version").propertyIndex().analyzed().ordered();
 489                     initRegexAll( idxBuilder.indexRule( FACET_NODE_TYPE ) );
 490                     idxBuilder.indexRule( MIXIN_META_SCM )
 491                         .property( "scm.connection" ).propertyIndex()
 492                         .property( "scm.developerConnection" ).propertyIndex()
 493                         .property( "scm.url").type("URI").propertyIndex().analyzed();
 494                     idxBuilder.indexRule( MIXIN_META_CI )
 495                         .property( "ci.system" ).propertyIndex( )
 496                         .property( "ci.ur" ).propertyIndex( ).analyzed( );
 497                     idxBuilder.indexRule( MIXIN_META_ISSUE )
 498                         .property( "issue.system").propertyIndex()
 499                         .property("issue.url").propertyIndex().analyzed();
 500                     idxBuilder.indexRule( MIXIN_META_ORGANIZATION )
 501                         .property( "org.name" ).propertyIndex( ).analyzed( )
 502                         .property( "org.url" ).propertyIndex( ).analyzed( );
 503                     idxBuilder.indexRule( MIXIN_META_LICENSE )
 504                         .property( "license.name" ).propertyIndex( ).analyzed( )
 505                         .property( "license.url" ).propertyIndex( ).analyzed( );
 506                     idxBuilder.indexRule( MIXIN_META_MAILINGLIST )
 507                         .property( "name" ).propertyIndex().analyzed();
 508                     initBaseRule(idxBuilder.indexRule( DEPENDENCY_NODE_TYPE ))
 509                         .property( "groupId" ).propertyIndex().analyzed().ordered()
 510                         .property( "artifactId").propertyIndex().analyzed().ordered()
 511                         .property("version").propertyIndex().analyzed().ordered()
 512                         .property("type").propertyIndex().analyzed().ordered()
 513                         .property( "classifier" ).propertyIndex().ordered()
 514                         .property("scope").propertyIndex()
 515                         .property("systemPath").propertyIndex().analyzed()
 516                         .property("optional").type("Boolean").propertyIndex();
 517
 518                     idxBuilder.aggregateRule( PROJECT_VERSION_NODE_TYPE ).include( "dependencies")
 519                         .path("dependencies/*" ).relativeNode();
 520
 521                     idxBuilder.build( );
 522
 523                     IndexUtils.createIndexDefinition( oakIdx, "baseIndexes", true, false, ImmutableList.of( "jcr:uuid", "rep:principalName" ), null );
 524
 525                     log.info( "Index: {} repo-lucene: {}", lucene, lucene.getChildNode( "repo-lucene" ) );
 526                     log.info( "repo-lucene Properties: {}", lucene.getChildNode( "repo-lucene" ).getProperties( ) );
 527                 } else {
 528
 529                     NodeBuilder lucene = oakIdx.child( "repo-lucene" );
 530                     lucene.setProperty( "reindex", true );
 531                     log.info( "No Index update" );
 532                 }
 533                 // IndexUtils.createIndexDefinition(  )
 534
 535             }
 536         };
 537
 538         //        ExternalObserverBuilder builder = new ExternalObserverBuilder(queue, tracker, statsProvider,
 539 //            executorService, queueSize);
 540 //        Observer observer = builder.build();
 541 //        builder.getBackgroundObserver();
 542
 543
 544
 545         log.info( "Starting Jcr repo with nodeStore {}", nodeStore );
 546         Jcr jcr = new Jcr( nodeStore ).with( editorProvider ) //
 547             .with( backgroundObserver ) //
 548             .with( externalIndexObserver )
 549             // .with(observer)
 550             .with( (QueryIndexProvider) indexProvider )
 551             .with (repoInitializer)
 552             .withAsyncIndexing( "async", 5 );
 553             //
 554             //.withAsyncIndexing( "async", 5 );
 555         StopWatch stopWatch = new StopWatch();
 556         stopWatch.start();
 557         Repository r = jcr.createRepository();
 558         stopWatch.stop();
 559         log.info( "time to create jcr repository: {} ms", stopWatch.getTime() );
 560 //        try
 561 //        {
 562 //            Thread.currentThread().sleep( 1000 );
 563 //        }
 564 //        catch ( InterruptedException e )
 565 //        {
 566 //            log.error( e.getMessage(), e );
 567 //        }
 568         return r;
 569
 570
 571     }
 572
 573     public void close()
 574     {
 575         log.info( "Closing JCR RepositoryFactory" );
 576         if ( fileStore != null )
 577         {
 578             fileStore.close();
 579         }
 580
 581         if (backgroundObserver != null){
 582             backgroundObserver.close();
 583         }
 584
 585         if (externalIndexObserver != null){
 586             externalIndexObserver.close();
 587         }
 588
 589         if (indexProvider != null) {
 590             indexProvider.close();
 591             indexProvider = null;
 592         }
 593
 594         if (documentQueue != null){
 595             try
 596             {
 597                 documentQueue.close();
 598             }
 599             catch ( IOException e )
 600             {
 601                 e.printStackTrace( );
 602             }
 603         }
 604
 605         if (nrtIndexFactory != null){
 606             try
 607             {
 608                 nrtIndexFactory.close();
 609             }
 610             catch ( IOException e )
 611             {
 612                 e.printStackTrace( );
 613             }
 614         }
 615
 616         //Close the copier first i.e. before executorService
 617         if (indexCopier != null){
 618             try
 619             {
 620                 indexCopier.close();
 621             }
 622             catch ( IOException e )
 623             {
 624                 e.printStackTrace( );
 625             }
 626         }
 627
 628         if (executorService != null){
 629             executorService.shutdown();
 630             try
 631             {
 632                 executorService.awaitTermination(1, TimeUnit.MINUTES);
 633             }
 634             catch ( InterruptedException e )
 635             {
 636                 e.printStackTrace( );
 637             }
 638         }
 639
 640         if (extractedTextCache != null) {
 641             extractedTextCache.close();
 642         }
 643
 644     }
 645
 646     public StoreType getStoreType()
 647     {
 648         return storeType;
 649     }
 650
 651     public void setStoreType( StoreType storeType )
 652     {
 653         this.storeType = storeType;
 654     }
 655
 656     public Path getRepositoryPath()
 657     {
 658         return repositoryPath;
 659     }
 660
 661     public void setRepositoryPath( Path repositoryPath )
 662     {
 663         this.repositoryPath = repositoryPath;
 664     }
 665
 666     public void setRepositoryPath( String repositoryPath )
 667     {
 668         this.repositoryPath = Paths.get( repositoryPath );
 669         if ( !Files.exists( this.repositoryPath ) )
 670         {
 671             try
 672             {
 673                 Files.createDirectories( this.repositoryPath );
 674             }
 675             catch ( IOException e )
 676             {
 677                 log.error( e.getMessage(), e );
 678                 throw new IllegalArgumentException( "cannot create directory:" + repositoryPath, e );
 679             }
 680         }
 681     }
 682
 683
 684 }