source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.metadata.repository.jcr;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.commons.lang3.time.StopWatch;
  23 import org.apache.jackrabbit.oak.api.Type;
  24 import org.apache.jackrabbit.oak.jcr.Jcr;
  25 import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
  26 import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoServiceImpl;
  27 import org.apache.jackrabbit.oak.plugins.index.IndexInfoProvider;
  28 import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
  29 import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl;
  30 import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
  31 import org.apache.jackrabbit.oak.plugins.index.aggregate.SimpleNodeAggregator;
  32 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexAugmentorFactory;
  33 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
  34 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
  35 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
  36 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexInfoProvider;
  37 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
  38 import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory;
  39 import org.apache.jackrabbit.oak.plugins.index.lucene.directory.BufferedOakDirectory;
  40 import org.apache.jackrabbit.oak.plugins.index.lucene.directory.LuceneIndexImporter;
  41 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
  42 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.ExternalObserverBuilder;
  43 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver;
  44 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory;
  45 import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner;
  46 import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
  47 import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory;
  48 import org.apache.jackrabbit.oak.plugins.index.lucene.score.impl.ScorerProviderFactoryImpl;
  49 import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
  50 import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
  51 import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
  52 import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
  53 import org.apache.jackrabbit.oak.plugins.name.Namespaces;
  54 import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
  55 import org.apache.jackrabbit.oak.segment.file.FileStore;
  56 import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
  57 import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
  58 import org.apache.jackrabbit.oak.spi.blob.FileBlobStore;
  59 import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
  60 import org.apache.jackrabbit.oak.spi.commit.BackgroundObserver;
  61 import org.apache.jackrabbit.oak.spi.commit.Observer;
  62 import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
  63 import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
  64 import org.apache.jackrabbit.oak.spi.mount.Mounts;
  65 import org.apache.jackrabbit.oak.spi.namespace.NamespaceConstants;
  66 import org.apache.jackrabbit.oak.spi.query.QueryIndex;
  67 import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
  68 import org.apache.jackrabbit.oak.spi.state.Clusterable;
  69 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
  70 import org.apache.jackrabbit.oak.spi.state.NodeStore;
  71 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
  72 import org.slf4j.Logger;
  73 import org.slf4j.LoggerFactory;
  74
  75 import javax.jcr.Repository;
  76 import java.io.Closeable;
  77 import java.io.IOException;
  78 import java.nio.file.Files;
  79 import java.nio.file.Path;
  80 import java.nio.file.Paths;
  81 import java.util.Arrays;
  82 import java.util.Collections;
  83 import java.util.HashSet;
  84 import java.util.List;
  85 import java.util.Set;
  86 import java.util.concurrent.ExecutorService;
  87 import java.util.concurrent.LinkedBlockingQueue;
  88 import java.util.concurrent.ThreadFactory;
  89 import java.util.concurrent.ThreadPoolExecutor;
  90 import java.util.concurrent.TimeUnit;
  91 import java.util.concurrent.atomic.AtomicInteger;
  92
  93 import static org.apache.archiva.metadata.repository.jcr.JcrConstants.*;
  94 import static org.apache.archiva.metadata.repository.jcr.OakRepositoryFactory.StoreType.IN_MEMORY_TYPE;
  95 import static org.apache.archiva.metadata.repository.jcr.OakRepositoryFactory.StoreType.SEGMENT_FILE_TYPE;
  96 import static org.apache.commons.io.FileUtils.ONE_MB;
  97 import static org.apache.jackrabbit.JcrConstants.*;
  98 import static org.apache.jackrabbit.oak.api.Type.NAME;
  99
 100 /**
 101  * Created by martin on 14.06.17.
 102  *
 103  * @author Martin Stockhammer
 104  * @since 3.0.0
 105  */
 106 public class OakRepositoryFactory
 107 {
 108
 109     private Logger log = LoggerFactory.getLogger( OakRepositoryFactory.class );
 110
 111     private FileStore fileStore;
 112
 113     private NodeStore nodeStore;
 114
 115     private IndexTracker tracker;
 116
 117     private DocumentQueue documentQueue;
 118
 119     private NRTIndexFactory nrtIndexFactory;
 120
 121     private IndexCopier indexCopier;
 122
 123     private ExecutorService executorService;
 124     private ExtractedTextCache extractedTextCache;
 125
 126     private boolean hybridIndex = true;
 127     private boolean prefetchEnabled = true;
 128     private boolean enableAsyncIndexOpen = true;
 129     int queueSize = 10000;
 130     int cleanerInterval = 10*60;
 131     boolean enableCopyOnWrite = true;
 132     boolean enableCopyOnRead = true;
 133     int cacheSizeInMB = 20;
 134     int cacheExpiryInSecs = 300;
 135     int threadPoolSize = 5;
 136
 137     private StatisticsProvider statisticsProvider;
 138
 139     private MountInfoProvider mountInfoProvider =  Mounts.defaultMountInfoProvider();
 140
 141     private AsyncIndexInfoService asyncIndexInfoService = null;
 142
 143     private LuceneIndexProvider indexProvider;
 144
 145     private ScorerProviderFactory scorerFactory = new ScorerProviderFactoryImpl( );
 146     private IndexAugmentorFactory augmentorFactory = new IndexAugmentorFactory( );
 147
 148     private ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollector activeDeletedBlobCollector = ActiveDeletedBlobCollectorFactory.NOOP;
 149
 150     private QueryIndex.NodeAggregator nodeAggregator = new SimpleNodeAggregator( );
 151
 152     private BackgroundObserver backgroundObserver;
 153
 154     private BackgroundObserver externalIndexObserver;
 155
 156     private GarbageCollectableBlobStore blobStore;
 157
 158     private PropertyIndexCleaner cleaner;
 159
 160     private IndexPathService indexPathService;
 161
 162     private LuceneIndexEditorProvider editorProvider;
 163
 164     private Path indexDir;
 165
 166     public enum StoreType
 167     {
 168         SEGMENT_FILE_TYPE,
 169         IN_MEMORY_TYPE;
 170     }
 171
 172     private StoreType storeType = SEGMENT_FILE_TYPE;
 173
 174     private Path repositoryPath = Paths.get( "repository" );
 175
 176     public OakRepositoryFactory() {
 177         final OakRepositoryFactory repositoryFactory = this;
 178         Runtime.getRuntime().addShutdownHook( new Thread( ( ) -> {
 179             if (repositoryFactory!=null)
 180             {
 181                 repositoryFactory.close( );
 182             }
 183         } ) );
 184     }
 185
 186     private void initializeExtractedTextCache( StatisticsProvider statisticsProvider) {
 187         boolean alwaysUsePreExtractedCache = false;
 188
 189         extractedTextCache = new ExtractedTextCache(
 190             cacheSizeInMB * ONE_MB,
 191             cacheExpiryInSecs,
 192             alwaysUsePreExtractedCache,
 193             indexDir.toFile(), statisticsProvider);
 194     }
 195
 196     private IndexTracker createTracker() throws IOException {
 197         IndexTracker tracker;
 198         if (enableCopyOnRead){
 199             initializeIndexCopier();
 200             log.info("Enabling CopyOnRead support. Index files would be copied under {}", indexDir.toAbsolutePath());
 201             if (hybridIndex) {
 202                 nrtIndexFactory = new NRTIndexFactory(indexCopier, statisticsProvider);
 203             }
 204             tracker = new IndexTracker(new DefaultIndexReaderFactory(mountInfoProvider, indexCopier), nrtIndexFactory);
 205         } else {
 206             tracker = new IndexTracker(new DefaultIndexReaderFactory(mountInfoProvider, null));
 207         }
 208
 209         tracker.setAsyncIndexInfoService(asyncIndexInfoService);
 210         tracker.refresh();
 211         return tracker;
 212     }
 213
 214     private void initializeIndexCopier() throws IOException {
 215         if(indexCopier != null){
 216             return;
 217         }
 218
 219         if (prefetchEnabled){
 220             log.info("Prefetching of index files enabled. Index would be opened after copying all new files locally");
 221         }
 222
 223         indexCopier = new IndexCopier(getExecutorService(), indexDir.toFile(), prefetchEnabled);
 224
 225     }
 226
 227     ExecutorService getExecutorService(){
 228         if (executorService == null){
 229             executorService = createExecutor();
 230         }
 231         return executorService;
 232     }
 233
 234     private ExecutorService createExecutor() {
 235         ThreadPoolExecutor executor = new ThreadPoolExecutor(threadPoolSize, threadPoolSize, 60L, TimeUnit.SECONDS,
 236             new LinkedBlockingQueue<Runnable>(), new ThreadFactory() {
 237             private final AtomicInteger counter = new AtomicInteger();
 238             private final Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
 239                 @Override
 240                 public void uncaughtException(Thread t, Throwable e) {
 241                     log.warn("Error occurred in asynchronous processing ", e);
 242                 }
 243             };
 244             @Override
 245             public Thread newThread(Runnable r) {
 246                 Thread thread = new Thread(r, createName());
 247                 thread.setDaemon(true);
 248                 thread.setPriority(Thread.MIN_PRIORITY);
 249                 thread.setUncaughtExceptionHandler(handler);
 250                 return thread;
 251             }
 252
 253             private String createName() {
 254                 return "oak-lucene-" + counter.getAndIncrement();
 255             }
 256         });
 257         executor.setKeepAliveTime(1, TimeUnit.MINUTES);
 258         executor.allowCoreThreadTimeOut(true);
 259         return executor;
 260     }
 261
 262     private void initialize(){
 263         if(indexProvider == null){
 264             return;
 265         }
 266
 267         if(nodeAggregator != null){
 268             log.debug("Using NodeAggregator {}", nodeAggregator.getClass());
 269         }
 270
 271         indexProvider.setAggregator(nodeAggregator);
 272     }
 273
 274     private void registerObserver() {
 275         Observer observer = indexProvider;
 276         if (enableAsyncIndexOpen) {
 277             backgroundObserver = new BackgroundObserver(indexProvider, getExecutorService(), 5);
 278             log.info("Registering the LuceneIndexProvider as a BackgroundObserver");
 279         }
 280     }
 281
 282     private void registerLocalIndexObserver(IndexTracker tracker) {
 283         if (!hybridIndex){
 284             log.info("Hybrid indexing feature disabled");
 285             return;
 286         }
 287         documentQueue = new DocumentQueue( queueSize, tracker, getExecutorService(), statisticsProvider);
 288         LocalIndexObserver localIndexObserver = new LocalIndexObserver(documentQueue, statisticsProvider);
 289
 290         int observerQueueSize = 1000;
 291         int builderMaxSize = 5000;
 292         // regs.add(bundleContext.registerService(JournalPropertyService.class.getName(),
 293         //    new LuceneJournalPropertyService(builderMaxSize), null));
 294         ExternalObserverBuilder builder = new ExternalObserverBuilder(documentQueue, tracker, statisticsProvider,
 295             getExecutorService(), observerQueueSize);
 296         log.info("Configured JournalPropertyBuilder with max size {} and backed by BackgroundObserver " +
 297             "with queue size {}", builderMaxSize, observerQueueSize);
 298
 299         Observer observer = builder.build();
 300         externalIndexObserver = builder.getBackgroundObserver();
 301         log.info("Hybrid indexing enabled for configured indexes with queue size of {}", queueSize );
 302     }
 303
 304     private IndexInfoProvider registerIndexInfoProvider() {
 305         return new LuceneIndexInfoProvider(nodeStore, asyncIndexInfoService, getIndexCheckDir().toFile());
 306     }
 307
 308     private Path getIndexCheckDir() {
 309         if (indexDir==null) {
 310             throw new NullPointerException( "Null value for indexDir encountered." );
 311         }
 312         return indexDir.resolve("indexCheckDir");
 313     }
 314
 315     private LuceneIndexImporter registerIndexImporterProvider() {
 316         return new LuceneIndexImporter(blobStore);
 317     }
 318
 319     private void registerPropertyIndexCleaner( ) {
 320
 321         if (cleanerInterval <= 0) {
 322             log.info("Property index cleaner would not be registered");
 323             return;
 324         }
 325
 326         cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService, statisticsProvider);
 327
 328         //Proxy check for DocumentNodeStore
 329         if (nodeStore instanceof Clusterable ) {
 330             cleaner.setRecursiveDelete(true);
 331             log.info("PropertyIndexCleaner configured to perform recursive delete");
 332         }
 333         log.info("Property index cleaner configured to run every [{}] seconds", cleanerInterval);
 334     }
 335
 336     private void registerIndexEditor( IndexTracker tracker) throws IOException {
 337         boolean enableCopyOnWrite = true;
 338         if (enableCopyOnWrite){
 339             initializeIndexCopier();
 340             editorProvider = new LuceneIndexEditorProvider(indexCopier, tracker, extractedTextCache,
 341                 augmentorFactory,  mountInfoProvider, activeDeletedBlobCollector, null, statisticsProvider);
 342             log.info("Enabling CopyOnWrite support. Index files would be copied under {}", indexDir.toAbsolutePath());
 343         } else {
 344             editorProvider = new LuceneIndexEditorProvider(null, tracker, extractedTextCache, augmentorFactory,
 345                 mountInfoProvider, activeDeletedBlobCollector, null, statisticsProvider);
 346         }
 347         editorProvider.setBlobStore(blobStore);
 348
 349         if (hybridIndex){
 350             if (documentQueue==null) {
 351                 throw new NullPointerException( "Null value for documentQueue encountered" );
 352             }
 353             editorProvider.setIndexingQueue(documentQueue);
 354         }
 355
 356
 357     }
 358
 359     public Repository createRepository()
 360         throws IOException, InvalidFileStoreVersionException
 361     {
 362
 363         indexDir = repositoryPath.resolve( ".index-lucene" );
 364         if (!Files.exists( indexDir )) {
 365             Files.createDirectories( indexDir );
 366         }
 367         blobStore = new FileBlobStore( indexDir.resolve( "blobs" ).toAbsolutePath().toString() );
 368
 369         statisticsProvider = StatisticsProvider.NOOP;
 370
 371         if ( SEGMENT_FILE_TYPE == storeType )
 372         {
 373             fileStore = FileStoreBuilder.fileStoreBuilder( repositoryPath.toFile() )
 374                 .withStatisticsProvider( statisticsProvider )
 375                 .build();
 376             nodeStore = SegmentNodeStoreBuilders.builder( fileStore ) //
 377                 .withStatisticsProvider( statisticsProvider ) //
 378                 .build();
 379         }
 380         else if ( IN_MEMORY_TYPE == storeType )
 381         {
 382             nodeStore = new MemoryNodeStore( );
 383         }
 384         else
 385         {
 386             throw new IllegalArgumentException( "Store type " + storeType + " not recognized" );
 387         }
 388
 389         asyncIndexInfoService = new AsyncIndexInfoServiceImpl( nodeStore );
 390
 391         indexPathService = new IndexPathServiceImpl( nodeStore, mountInfoProvider );
 392
 393         BufferedOakDirectory.setEnableWritingSingleBlobIndexFile( true );
 394
 395         initializeExtractedTextCache( statisticsProvider );
 396
 397         tracker = createTracker();
 398
 399         indexProvider = new LuceneIndexProvider(tracker, scorerFactory, augmentorFactory);
 400
 401         initialize();
 402         registerObserver();
 403         registerLocalIndexObserver(tracker);
 404         registerIndexInfoProvider();
 405         registerIndexImporterProvider();
 406         registerPropertyIndexCleaner();
 407
 408         registerIndexEditor(tracker);
 409
 410
 411
 412         RepositoryInitializer repoInitializer = new RepositoryInitializer( )
 413         {
 414             private IndexDefinitionBuilder.PropertyRule initRegexAll( IndexDefinitionBuilder.IndexRule rule ) {
 415                 return rule
 416                     .indexNodeName( )
 417                     .property(JCR_LASTMODIFIED ).propertyIndex().type( "Date" ).ordered()
 418                     .property(JCR_PRIMARYTYPE).propertyIndex()
 419                     .property(JCR_MIXINTYPES).propertyIndex()
 420                     .property(JCR_PATH).propertyIndex().ordered()
 421                     .property( FulltextIndexConstants.REGEX_ALL_PROPS, true )
 422                     .propertyIndex().analyzed( ).nodeScopeIndex();
 423             }
 424
 425             private IndexDefinitionBuilder.PropertyRule initBaseRule( IndexDefinitionBuilder.IndexRule rule ) {
 426                 return rule
 427                     .indexNodeName( )
 428                     .property(JCR_CREATED).propertyIndex().type("Date").ordered()
 429                     .property(JCR_LASTMODIFIED ).propertyIndex().type( "Date" ).ordered()
 430                     .property(JCR_PRIMARYTYPE).propertyIndex()
 431                     .property(JCR_MIXINTYPES).propertyIndex()
 432                     .property(JCR_PATH).propertyIndex().ordered()
 433                     .property( "id" ).propertyIndex().analyzed( );
 434             }
 435
 436             @Override
 437             public void initialize(  NodeBuilder root )
 438             {
 439                 NodeBuilder namespaces;
 440                 if ( !root.hasChildNode( NamespaceConstants.REP_NAMESPACES ) )
 441                 {
 442                     namespaces = Namespaces.createStandardMappings( root );
 443                     Namespaces.buildIndexNode( namespaces ); // index node for faster lookup
 444                 }
 445                 else
 446                 {
 447                     namespaces = root.getChildNode( NamespaceConstants.REP_NAMESPACES );
 448                 }
 449                 Namespaces.addCustomMapping( namespaces, "http://archiva.apache.org/jcr/", "archiva" );
 450
 451                 log.info( "Creating index " );
 452
 453                 NodeBuilder oakIdx = IndexUtils.getOrCreateOakIndex( root );
 454                 if (!oakIdx.hasChildNode( "repo-lucene" ))
 455                 {
 456                     NodeBuilder lucene = oakIdx.child( "repo-lucene" );
 457                     lucene.setProperty( JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", NAME );
 458
 459                     lucene.setProperty( "compatVersion", 2 );
 460                     lucene.setProperty( "type", "lucene" );
 461                     // lucene.setProperty("async", "async");
 462                     // lucene.setProperty( INCLUDE_PROPERTY_TYPES, ImmutableSet.of(  ), Type.STRINGS );
 463                     // lucene.setProperty("refresh",true);
 464                     NodeBuilder rules = lucene.child( "indexRules" ).
 465                         setProperty( JCR_PRIMARYTYPE, NT_UNSTRUCTURED, NAME );
 466                     Set<String> parameterSet = new HashSet<>( Arrays.asList(REPOSITORY_NODE_TYPE,
 467                         NAMESPACE_MIXIN_TYPE, //
 468                         PROJECT_MIXIN_TYPE,
 469                         PROJECT_VERSION_NODE_TYPE, //
 470                         ARTIFACT_NODE_TYPE, //
 471                         FACET_NODE_TYPE //
 472                         ));
 473                     parameterSet = Collections.unmodifiableSet(parameterSet);
 474                     rules.setProperty( ":childOrder", parameterSet, Type.STRINGS );
 475                     IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder( lucene );
 476                     idxBuilder.async( "async", "nrt", "sync" ).includedPaths( "/repositories" ).evaluatePathRestrictions();
 477
 478                     initBaseRule(idxBuilder.indexRule( REPOSITORY_NODE_TYPE ));
 479                     initBaseRule(idxBuilder.indexRule(NAMESPACE_MIXIN_TYPE))
 480                         .property( "namespace" ).propertyIndex().analyzed();
 481                     initBaseRule(idxBuilder.indexRule(PROJECT_MIXIN_TYPE))
 482                         .property( "name" ).propertyIndex().analyzed().notNullCheckEnabled().nullCheckEnabled();
 483                     initBaseRule( idxBuilder.indexRule( PROJECT_VERSION_NODE_TYPE ) )
 484                         .property("name").propertyIndex().analyzed().notNullCheckEnabled().nullCheckEnabled()
 485                         .property("description").propertyIndex().analyzed().notNullCheckEnabled().nullCheckEnabled()
 486                         .property("url").propertyIndex().analyzed( ).notNullCheckEnabled().nullCheckEnabled()
 487                         .property("incomplete").type("Boolean").propertyIndex()
 488                         .property("mailinglist/name").propertyIndex().analyzed()
 489                         .property("license/license.name").propertyIndex().analyzed();
 490                     initBaseRule(idxBuilder.indexRule( ARTIFACT_NODE_TYPE ))
 491                         .property( "whenGathered" ).type("Date").propertyIndex().analyzed().ordered()
 492                         .property("size").type("Long").propertyIndex().analyzed().ordered()
 493                         .property("version").propertyIndex().analyzed().ordered()
 494                         .property("checksums/*/value").propertyIndex();
 495
 496                     initBaseRule( idxBuilder.indexRule( CHECKSUM_NODE_TYPE ) )
 497                         .property("type").propertyIndex()
 498                         .property("value").propertyIndex();
 499
 500                     initRegexAll( idxBuilder.indexRule( FACET_NODE_TYPE ) )
 501                         .property("archiva:facetId").propertyIndex().analyzed().ordered()
 502                         .property("archiva:name").propertyIndex().analyzed().ordered().nullCheckEnabled().notNullCheckEnabled();
 503
 504                     idxBuilder.indexRule( MIXIN_META_SCM )
 505                         .property( "scm.connection" ).propertyIndex()
 506                         .property( "scm.developerConnection" ).propertyIndex()
 507                         .property( "scm.url").type("URI").propertyIndex().analyzed();
 508                     idxBuilder.indexRule( MIXIN_META_CI )
 509                         .property( "ci.system" ).propertyIndex( )
 510                         .property( "ci.ur" ).propertyIndex( ).analyzed( );
 511                     idxBuilder.indexRule( MIXIN_META_ISSUE )
 512                         .property( "issue.system").propertyIndex()
 513                         .property("issue.url").propertyIndex().analyzed();
 514                     idxBuilder.indexRule( MIXIN_META_ORGANIZATION )
 515                         .property( "org.name" ).propertyIndex( ).analyzed( )
 516                         .property( "org.url" ).propertyIndex( ).analyzed( );
 517                     idxBuilder.indexRule( LICENSE_NODE_TYPE )
 518                         .property( "license.name" ).propertyIndex( ).analyzed( )
 519                         .property( "license.url" ).propertyIndex( ).analyzed( );
 520                     idxBuilder.indexRule( MAILINGLIST_NODE_TYPE )
 521                         .property( "name" ).propertyIndex().analyzed();
 522                     initBaseRule(idxBuilder.indexRule( DEPENDENCY_NODE_TYPE ))
 523                         .property( "groupId" ).propertyIndex().analyzed().ordered()
 524                         .property( "artifactId").propertyIndex().analyzed().ordered()
 525                         .property("version").propertyIndex().analyzed().ordered()
 526                         .property("type").propertyIndex().analyzed().ordered()
 527                         .property( "classifier" ).propertyIndex().ordered()
 528                         .property("scope").propertyIndex()
 529                         .property("systemPath").propertyIndex().analyzed()
 530                         .property("optional").type("Boolean").propertyIndex();
 531
 532                     idxBuilder.aggregateRule( PROJECT_VERSION_NODE_TYPE ).include( "dependencies")
 533                         .path("dependencies/*" ).relativeNode();
 534
 535                     idxBuilder.build( );
 536
 537                     List<String> parameterList = Arrays.asList("jcr:uuid", "rep:principalName");
 538                     parameterList = Collections.unmodifiableList(parameterList);
 539                     IndexUtils.createIndexDefinition( oakIdx, "baseIndexes", true, false, parameterList, null );
 540
 541                     log.info( "Index: {} repo-lucene: {}", lucene, lucene.getChildNode( "repo-lucene" ) );
 542                     log.info( "repo-lucene Properties: {}", lucene.getChildNode( "repo-lucene" ).getProperties( ) );
 543                 } else {
 544                     log.info( "No Index update" );
 545                 }
 546                 // IndexUtils.createIndexDefinition(  )
 547
 548             }
 549         };
 550
 551         //        ExternalObserverBuilder builder = new ExternalObserverBuilder(queue, tracker, statsProvider,
 552 //            executorService, queueSize);
 553 //        Observer observer = builder.build();
 554 //        builder.getBackgroundObserver();
 555
 556
 557
 558         log.info( "Starting Jcr repo with nodeStore {}", nodeStore );
 559         Jcr jcr = new Jcr( nodeStore ).with( editorProvider ) //
 560             .with( backgroundObserver ) //
 561             .with( externalIndexObserver )
 562             // .with(observer)
 563             .with( (QueryIndexProvider) indexProvider )
 564             .with (repoInitializer)
 565             .withAsyncIndexing( "async", 5 );
 566             //
 567             //.withAsyncIndexing( "async", 5 );
 568         StopWatch stopWatch = new StopWatch();
 569         stopWatch.start();
 570         Repository r = jcr.createRepository();
 571         stopWatch.stop();
 572         log.info( "time to create jcr repository: {} ms", stopWatch.getTime() );
 573
 574         return r;
 575
 576
 577     }
 578
 579     private void closeSilently( Closeable service) {
 580         if (service!=null) {
 581             try
 582             {
 583                 service.close();
 584             }
 585             catch ( Throwable e )
 586             {
 587                 //
 588             }
 589         }
 590     }
 591
 592     public void close()
 593     {
 594         log.info( "Closing JCR RepositoryFactory" );
 595         closeSilently( fileStore );
 596         closeSilently( backgroundObserver );
 597         closeSilently( externalIndexObserver );
 598         closeSilently( indexProvider );
 599         indexProvider = null;
 600         closeSilently( documentQueue );
 601         closeSilently( nrtIndexFactory );
 602         closeSilently( indexCopier );
 603
 604         if (executorService != null){
 605             executorService.shutdown();
 606             try
 607             {
 608                 executorService.awaitTermination(1, TimeUnit.MINUTES);
 609             }
 610             catch ( InterruptedException e )
 611             {
 612                 e.printStackTrace( );
 613             }
 614         }
 615
 616         if (extractedTextCache != null) {
 617             extractedTextCache.close();
 618         }
 619
 620     }
 621
 622     public StoreType getStoreType()
 623     {
 624         return storeType;
 625     }
 626
 627     public void setStoreType( StoreType storeType )
 628     {
 629         this.storeType = storeType;
 630     }
 631
 632     public Path getRepositoryPath()
 633     {
 634         return repositoryPath;
 635     }
 636
 637     public void setRepositoryPath( Path repositoryPath )
 638     {
 639         this.repositoryPath = repositoryPath;
 640     }
 641
 642     public void setRepositoryPath( String repositoryPath )
 643     {
 644         this.repositoryPath = Paths.get( repositoryPath );
 645         if ( !Files.exists( this.repositoryPath ) )
 646         {
 647             try
 648             {
 649                 Files.createDirectories( this.repositoryPath );
 650             }
 651             catch ( IOException e )
 652             {
 653                 log.error( e.getMessage(), e );
 654                 throw new IllegalArgumentException( "cannot create directory:" + repositoryPath, e );
 655             }
 656         }
 657     }
 658
 659
 660 }