source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.metadata.repository.jcr;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import com.google.common.collect.ImmutableSet;
  23 import org.apache.commons.lang.time.StopWatch;
  24 import org.apache.jackrabbit.JcrConstants;
  25 import org.apache.jackrabbit.oak.Oak;
  26 import org.apache.jackrabbit.oak.api.Type;
  27 import org.apache.jackrabbit.oak.jcr.Jcr;
  28 import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
  29 import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
  30 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
  31 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
  32 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
  33 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
  34 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
  35 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver;
  36 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory;
  37 import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
  38 import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
  39 import org.apache.jackrabbit.oak.segment.file.FileStore;
  40 import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
  41 import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
  42 import org.apache.jackrabbit.oak.spi.commit.Observer;
  43 import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
  44 import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
  45 import org.apache.jackrabbit.oak.spi.mount.Mounts;
  46 import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
  47 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
  48 import org.apache.jackrabbit.oak.spi.state.NodeStore;
  49 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
  50 import org.slf4j.Logger;
  51 import org.slf4j.LoggerFactory;
  52
  53 import javax.annotation.Nonnull;
  54 import javax.jcr.Repository;
  55 import java.io.File;
  56 import java.io.IOException;
  57 import java.nio.file.Files;
  58 import java.nio.file.Path;
  59 import java.nio.file.Paths;
  60 import java.util.concurrent.ExecutorService;
  61 import java.util.concurrent.Executors;
  62
  63 import static org.apache.archiva.metadata.repository.jcr.RepositoryFactory.StoreType.IN_MEMORY_TYPE;
  64 import static org.apache.archiva.metadata.repository.jcr.RepositoryFactory.StoreType.SEGMENT_FILE_TYPE;
  65 import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
  66
  67 /**
  68  * Created by martin on 14.06.17.
  69  *
  70  * @author Martin Stockhammer
  71  * @since 3.0.0
  72  */
  73 public class RepositoryFactory
  74 {
  75
  76     private Logger log = LoggerFactory.getLogger( RepositoryFactory.class );
  77
  78     private FileStore fileStore;
  79
  80     public enum StoreType
  81     {
  82         SEGMENT_FILE_TYPE,
  83         IN_MEMORY_TYPE;
  84     }
  85
  86
  87     private StoreType storeType = SEGMENT_FILE_TYPE;
  88
  89     Path repositoryPath = Paths.get( "repository" );
  90
  91     public Repository createRepository()
  92         throws IOException, InvalidFileStoreVersionException
  93     {
  94         NodeStore nodeStore;
  95         if ( SEGMENT_FILE_TYPE == storeType )
  96         {
  97             fileStore = FileStoreBuilder.fileStoreBuilder( repositoryPath.toFile() ).build();
  98             nodeStore = SegmentNodeStoreBuilders.builder( fileStore ).build();
  99         }
 100         else if ( IN_MEMORY_TYPE == storeType )
 101         {
 102             nodeStore = null;
 103         }
 104         else
 105         {
 106             throw new IllegalArgumentException( "Store type " + storeType + " not recognized" );
 107         }
 108
 109         Oak oak = nodeStore == null ? new Oak() : new Oak( nodeStore );
 110         oak.with( new RepositoryInitializer()
 111         {
 112             @Override
 113             public void initialize( @Nonnull NodeBuilder root )
 114             {
 115                 log.info( "Creating index " );
 116
 117                 NodeBuilder lucene = IndexUtils.getOrCreateOakIndex( root ).child( "lucene" );
 118                 lucene.setProperty( JcrConstants.JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", Type.NAME );
 119
 120                 lucene.setProperty( "compatVersion", 2 );
 121                 lucene.setProperty( "type", "lucene" );
 122                 // lucene.setProperty("async", "async");
 123                 lucene.setProperty( INCLUDE_PROPERTY_TYPES, ImmutableSet.of( "String" ), Type.STRINGS );
 124                 // lucene.setProperty("refresh",true);
 125                 lucene.setProperty( "async", ImmutableSet.of( "async", "sync" ), Type.STRINGS );
 126                 NodeBuilder rules = lucene.child( "indexRules" ).
 127                     setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 128                 rules.setProperty( ":childOrder", ImmutableSet.of( "archiva:projectVersion", //
 129                                                                    "archiva:artifact", //
 130                                                                    "archiva:facet", //
 131                                                                    "archiva:namespace", //
 132                                                                    "archiva:project" ), //
 133                                    Type.STRINGS );
 134                 NodeBuilder allProps = rules.child( "archiva:projectVersion" ) //
 135                     .child( "properties" ) //
 136                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 137                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 138                     .setProperty( "indexNodeName", true ) //
 139                     .child( "allProps" ) //
 140                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 141                 allProps.setProperty( "name", ".*" );
 142                 allProps.setProperty( "isRegexp", true );
 143                 allProps.setProperty( "nodeScopeIndex", true );
 144                 allProps.setProperty( "index", true );
 145                 allProps.setProperty( "analyzed", true );
 146                 // allProps.setProperty("propertyIndex",true);
 147                 allProps = rules.child( "archiva:artifact" ) //
 148                     .child( "properties" ) //
 149                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 150                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 151                     .setProperty( "indexNodeName", true ).child( "allProps" ) //
 152                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 153                 allProps.setProperty( "name", ".*" );
 154                 allProps.setProperty( "isRegexp", true );
 155                 allProps.setProperty( "nodeScopeIndex", true );
 156                 allProps.setProperty( "index", true );
 157                 allProps.setProperty( "analyzed", true );
 158                 allProps = rules.child( "archiva:facet" ) //
 159                     .child( "properties" ) //
 160                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 161                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 162                     .setProperty( "indexNodeName", true ) //
 163                     .child( "allProps" ) //
 164                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 165                 allProps.setProperty( "name", ".*" );
 166                 allProps.setProperty( "isRegexp", true );
 167                 allProps.setProperty( "nodeScopeIndex", true );
 168                 allProps.setProperty( "index", true );
 169                 allProps.setProperty( "analyzed", true );
 170                 allProps = rules.child( "archiva:namespace" ) //
 171                     .child( "properties" ) //
 172                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 173                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 174                     .setProperty( "indexNodeName", true ) //
 175                     .child( "allProps" ) //
 176                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 177                 allProps.setProperty( "name", ".*" );
 178                 allProps.setProperty( "isRegexp", true );
 179                 allProps.setProperty( "nodeScopeIndex", true );
 180                 allProps.setProperty( "index", true );
 181                 allProps.setProperty( "analyzed", true );
 182                 allProps = rules.child( "archiva:project" ) //
 183                     .child( "properties" ) //
 184                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 185                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 186                     .setProperty( "indexNodeName", true ) //
 187                     .child( "allProps" ) //
 188                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 189                 allProps.setProperty( "name", ".*" );
 190                 allProps.setProperty( "isRegexp", true );
 191                 allProps.setProperty( "nodeScopeIndex", true );
 192                 allProps.setProperty( "index", true );
 193                 allProps.setProperty( "analyzed", true );
 194
 195                 log.info( "Index: {} myIndex {}", lucene, lucene.getChildNode( "myIndex" ) );
 196                 log.info( "myIndex {}", lucene.getChildNode( "myIndex" ).getProperties() );
 197                 // IndexUtils.createIndexDefinition(  )
 198
 199             }
 200         } );
 201
 202         ExecutorService executorService = createExecutor();
 203         StatisticsProvider statsProvider = StatisticsProvider.NOOP;
 204         int queueSize = Integer.getInteger( "queueSize", 10000 );
 205         File indexDir = Files.createTempDirectory( "archiva_index" ).toFile();
 206         log.info( "Queue Index {}", indexDir.toString() );
 207         IndexCopier indexCopier = new IndexCopier( executorService, indexDir, true );
 208         NRTIndexFactory nrtIndexFactory = new NRTIndexFactory( indexCopier, statsProvider );
 209         MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider();
 210         IndexTracker tracker =
 211             new IndexTracker( new DefaultIndexReaderFactory( mountInfoProvider, indexCopier ), nrtIndexFactory );
 212         DocumentQueue queue = new DocumentQueue( queueSize, tracker, executorService, statsProvider );
 213         LocalIndexObserver localIndexObserver = new LocalIndexObserver( queue, statsProvider );
 214         LuceneIndexProvider provider = new LuceneIndexProvider( tracker );
 215
 216         //        ExternalObserverBuilder builder = new ExternalObserverBuilder(queue, tracker, statsProvider,
 217 //            executorService, queueSize);
 218 //        Observer observer = builder.build();
 219 //        builder.getBackgroundObserver();
 220
 221         LuceneIndexEditorProvider editorProvider = //
 222             new LuceneIndexEditorProvider( null, tracker, //
 223                                            new ExtractedTextCache( 0, 0 ), //
 224                                            null, mountInfoProvider );
 225         editorProvider.setIndexingQueue( queue );
 226
 227         log.info( "Oak: {} with nodeStore {}", oak, nodeStore );
 228         Jcr jcr = new Jcr( oak ).with( editorProvider ) //
 229             .with( (Observer) provider ) //
 230             .with( localIndexObserver )
 231             // .with(observer)
 232             .with( (QueryIndexProvider) provider ); //
 233             //.withAsyncIndexing( "async", 5 );
 234         StopWatch stopWatch = new StopWatch();
 235         stopWatch.start();
 236         Repository r = jcr.createRepository();
 237         stopWatch.stop();
 238         log.info( "time to create jcr repository: {} ms", stopWatch.getTime() );
 239 //        try
 240 //        {
 241 //            Thread.currentThread().sleep( 1000 );
 242 //        }
 243 //        catch ( InterruptedException e )
 244 //        {
 245 //            log.error( e.getMessage(), e );
 246 //        }
 247         return r;
 248
 249
 250     }
 251
 252     public void close()
 253     {
 254         if ( fileStore != null )
 255         {
 256             fileStore.close();
 257         }
 258     }
 259
 260     public StoreType getStoreType()
 261     {
 262         return storeType;
 263     }
 264
 265     public void setStoreType( StoreType storeType )
 266     {
 267         this.storeType = storeType;
 268     }
 269
 270     public Path getRepositoryPath()
 271     {
 272         return repositoryPath;
 273     }
 274
 275     public void setRepositoryPath( Path repositoryPath )
 276     {
 277         this.repositoryPath = repositoryPath;
 278     }
 279
 280     public void setRepositoryPath( String repositoryPath )
 281     {
 282         this.repositoryPath = Paths.get( repositoryPath );
 283         if ( !Files.exists( this.repositoryPath ) )
 284         {
 285             try
 286             {
 287                 Files.createDirectories( this.repositoryPath );
 288             }
 289             catch ( IOException e )
 290             {
 291                 log.error( e.getMessage(), e );
 292                 throw new IllegalArgumentException( "cannot create directory:" + repositoryPath, e );
 293             }
 294         }
 295     }
 296
 297     private ExecutorService createExecutor()
 298     {
 299         return Executors.newCachedThreadPool();
 300 //
 301 //        ThreadPoolExecutor executor =
 302 //            new ThreadPoolExecutor( 0, 5, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(),
 303 //                                    new ThreadFactory()
 304 //                                    {
 305 //                                        private final AtomicInteger counter = new AtomicInteger();
 306 //
 307 //                                        private final Thread.UncaughtExceptionHandler handler =
 308 //                                            new Thread.UncaughtExceptionHandler()
 309 //                                            {
 310 //                                                @Override
 311 //                                                public void uncaughtException( Thread t, Throwable e )
 312 //                                                {
 313 //                                                    log.warn( "Error occurred in asynchronous processing ", e );
 314 //                                                }
 315 //                                            };
 316 //
 317 //                                        @Override
 318 //                                        public Thread newThread( @Nonnull Runnable r )
 319 //                                        {
 320 //                                            Thread thread = new Thread( r, createName() );
 321 //                                            thread.setDaemon( true );
 322 //                                            thread.setPriority( Thread.MIN_PRIORITY );
 323 //                                            thread.setUncaughtExceptionHandler( handler );
 324 //                                            return thread;
 325 //                                        }
 326 //
 327 //                                        private String createName()
 328 //                                        {
 329 //                                            return "oak-lucene-" + counter.getAndIncrement();
 330 //                                        }
 331 //                                    } );
 332 //        executor.setKeepAliveTime( 1, TimeUnit.MINUTES );
 333 //        executor.allowCoreThreadTimeOut( true );
 334 //        return executor;
 335     }
 336
 337 }