source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.metadata.repository.jcr;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import com.google.common.collect.ImmutableSet;
  23 import org.apache.commons.lang.time.StopWatch;
  24 import org.apache.jackrabbit.JcrConstants;
  25 import org.apache.jackrabbit.oak.Oak;
  26 import org.apache.jackrabbit.oak.api.Type;
  27 import org.apache.jackrabbit.oak.jcr.Jcr;
  28 import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
  29 import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
  30 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
  31 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
  32 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
  33 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
  34 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
  35 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver;
  36 import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory;
  37 import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
  38 import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
  39 import org.apache.jackrabbit.oak.segment.file.FileStore;
  40 import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
  41 import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
  42 import org.apache.jackrabbit.oak.spi.commit.Observer;
  43 import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
  44 import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
  45 import org.apache.jackrabbit.oak.spi.mount.Mounts;
  46 import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
  47 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
  48 import org.apache.jackrabbit.oak.spi.state.NodeStore;
  49 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
  50 import org.slf4j.Logger;
  51 import org.slf4j.LoggerFactory;
  52
  53 import javax.annotation.Nonnull;
  54 import javax.jcr.Repository;
  55 import java.io.File;
  56 import java.io.IOException;
  57 import java.nio.file.Files;
  58 import java.nio.file.Path;
  59 import java.nio.file.Paths;
  60 import java.util.concurrent.ExecutorService;
  61 import java.util.concurrent.Executors;
  62
  63 import static org.apache.archiva.metadata.repository.jcr.RepositoryFactory.StoreType.IN_MEMORY_TYPE;
  64 import static org.apache.archiva.metadata.repository.jcr.RepositoryFactory.StoreType.SEGMENT_FILE_TYPE;
  65 import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
  66
  67 /**
  68  * Created by martin on 14.06.17.
  69  *
  70  * @author Martin Stockhammer
  71  * @since 3.0.0
  72  */
  73 public class RepositoryFactory
  74 {
  75
  76     private Logger log = LoggerFactory.getLogger( RepositoryFactory.class );
  77
  78     private FileStore fileStore;
  79
  80     private ExecutorService executorService;
  81
  82     public enum StoreType
  83     {
  84         SEGMENT_FILE_TYPE,
  85         IN_MEMORY_TYPE;
  86     }
  87
  88     private StoreType storeType = SEGMENT_FILE_TYPE;
  89
  90     private Path repositoryPath = Paths.get( "repository" );
  91
  92     public Repository createRepository()
  93         throws IOException, InvalidFileStoreVersionException
  94     {
  95         createExecutor();
  96         NodeStore nodeStore;
  97         if ( SEGMENT_FILE_TYPE == storeType )
  98         {
  99             fileStore = FileStoreBuilder.fileStoreBuilder( repositoryPath.toFile() ).build();
 100             nodeStore = SegmentNodeStoreBuilders.builder( fileStore ).build();
 101         }
 102         else if ( IN_MEMORY_TYPE == storeType )
 103         {
 104             nodeStore = null;
 105         }
 106         else
 107         {
 108             throw new IllegalArgumentException( "Store type " + storeType + " not recognized" );
 109         }
 110
 111         Oak oak = nodeStore == null ? new Oak() : new Oak( nodeStore );
 112         oak.with( new RepositoryInitializer()
 113         {
 114             @Override
 115             public void initialize( @Nonnull NodeBuilder root )
 116             {
 117                 log.info( "Creating index " );
 118
 119                 NodeBuilder lucene = IndexUtils.getOrCreateOakIndex( root ).child( "lucene" );
 120                 lucene.setProperty( JcrConstants.JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", Type.NAME );
 121
 122                 lucene.setProperty( "compatVersion", 2 );
 123                 lucene.setProperty( "type", "lucene" );
 124                 // lucene.setProperty("async", "async");
 125                 lucene.setProperty( INCLUDE_PROPERTY_TYPES, ImmutableSet.of( "String" ), Type.STRINGS );
 126                 // lucene.setProperty("refresh",true);
 127                 lucene.setProperty( "async", ImmutableSet.of( "async", "sync" ), Type.STRINGS );
 128                 NodeBuilder rules = lucene.child( "indexRules" ).
 129                     setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 130                 rules.setProperty( ":childOrder", ImmutableSet.of( "archiva:projectVersion", //
 131                                                                    "archiva:artifact", //
 132                                                                    "archiva:facet", //
 133                                                                    "archiva:namespace", //
 134                                                                    "archiva:project" ), //
 135                                    Type.STRINGS );
 136                 NodeBuilder allProps = rules.child( "archiva:projectVersion" ) //
 137                     .child( "properties" ) //
 138                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 139                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 140                     .setProperty( "indexNodeName", true ) //
 141                     .child( "allProps" ) //
 142                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 143                 allProps.setProperty( "name", ".*" );
 144                 allProps.setProperty( "isRegexp", true );
 145                 allProps.setProperty( "nodeScopeIndex", true );
 146                 allProps.setProperty( "index", true );
 147                 allProps.setProperty( "analyzed", true );
 148                 // allProps.setProperty("propertyIndex",true);
 149                 allProps = rules.child( "archiva:artifact" ) //
 150                     .child( "properties" ) //
 151                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 152                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 153                     .setProperty( "indexNodeName", true ).child( "allProps" ) //
 154                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 155                 allProps.setProperty( "name", ".*" );
 156                 allProps.setProperty( "isRegexp", true );
 157                 allProps.setProperty( "nodeScopeIndex", true );
 158                 allProps.setProperty( "index", true );
 159                 allProps.setProperty( "analyzed", true );
 160                 allProps = rules.child( "archiva:facet" ) //
 161                     .child( "properties" ) //
 162                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 163                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 164                     .setProperty( "indexNodeName", true ) //
 165                     .child( "allProps" ) //
 166                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 167                 allProps.setProperty( "name", ".*" );
 168                 allProps.setProperty( "isRegexp", true );
 169                 allProps.setProperty( "nodeScopeIndex", true );
 170                 allProps.setProperty( "index", true );
 171                 allProps.setProperty( "analyzed", true );
 172                 allProps = rules.child( "archiva:namespace" ) //
 173                     .child( "properties" ) //
 174                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 175                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 176                     .setProperty( "indexNodeName", true ) //
 177                     .child( "allProps" ) //
 178                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 179                 allProps.setProperty( "name", ".*" );
 180                 allProps.setProperty( "isRegexp", true );
 181                 allProps.setProperty( "nodeScopeIndex", true );
 182                 allProps.setProperty( "index", true );
 183                 allProps.setProperty( "analyzed", true );
 184                 allProps = rules.child( "archiva:project" ) //
 185                     .child( "properties" ) //
 186                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
 187                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
 188                     .setProperty( "indexNodeName", true ) //
 189                     .child( "allProps" ) //
 190                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
 191                 allProps.setProperty( "name", ".*" );
 192                 allProps.setProperty( "isRegexp", true );
 193                 allProps.setProperty( "nodeScopeIndex", true );
 194                 allProps.setProperty( "index", true );
 195                 allProps.setProperty( "analyzed", true );
 196
 197                 log.info( "Index: {} myIndex {}", lucene, lucene.getChildNode( "myIndex" ) );
 198                 log.info( "myIndex {}", lucene.getChildNode( "myIndex" ).getProperties() );
 199                 // IndexUtils.createIndexDefinition(  )
 200
 201             }
 202         } );
 203
 204         StatisticsProvider statsProvider = StatisticsProvider.NOOP;
 205         int queueSize = Integer.getInteger( "queueSize", 10000 );
 206         File indexDir = Files.createTempDirectory( "archiva_index" ).toFile();
 207         log.info( "Queue Index {}", indexDir.toString() );
 208         IndexCopier indexCopier = new IndexCopier( executorService, indexDir, true );
 209         NRTIndexFactory nrtIndexFactory = new NRTIndexFactory( indexCopier, statsProvider );
 210         MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider();
 211         IndexTracker tracker =
 212             new IndexTracker( new DefaultIndexReaderFactory( mountInfoProvider, indexCopier ), nrtIndexFactory );
 213         DocumentQueue queue = new DocumentQueue( queueSize, tracker, executorService, statsProvider );
 214         LocalIndexObserver localIndexObserver = new LocalIndexObserver( queue, statsProvider );
 215         LuceneIndexProvider provider = new LuceneIndexProvider( tracker );
 216
 217         //        ExternalObserverBuilder builder = new ExternalObserverBuilder(queue, tracker, statsProvider,
 218 //            executorService, queueSize);
 219 //        Observer observer = builder.build();
 220 //        builder.getBackgroundObserver();
 221
 222         LuceneIndexEditorProvider editorProvider = //
 223             new LuceneIndexEditorProvider( null, tracker, //
 224                                            new ExtractedTextCache( 0, 0 ), //
 225                                            null, mountInfoProvider );
 226         editorProvider.setIndexingQueue( queue );
 227
 228         log.info( "Oak: {} with nodeStore {}", oak, nodeStore );
 229         Jcr jcr = new Jcr( oak ).with( editorProvider ) //
 230             .with( (Observer) provider ) //
 231             .with( localIndexObserver )
 232             // .with(observer)
 233             .with( (QueryIndexProvider) provider ); //
 234             //.withAsyncIndexing( "async", 5 );
 235         StopWatch stopWatch = new StopWatch();
 236         stopWatch.start();
 237         Repository r = jcr.createRepository();
 238         stopWatch.stop();
 239         log.info( "time to create jcr repository: {} ms", stopWatch.getTime() );
 240 //        try
 241 //        {
 242 //            Thread.currentThread().sleep( 1000 );
 243 //        }
 244 //        catch ( InterruptedException e )
 245 //        {
 246 //            log.error( e.getMessage(), e );
 247 //        }
 248         return r;
 249
 250
 251     }
 252
 253     public void close()
 254     {
 255         if ( fileStore != null )
 256         {
 257             fileStore.close();
 258         }
 259         if (executorService != null)
 260         {
 261             executorService.shutdownNow();
 262         }
 263     }
 264
 265     public StoreType getStoreType()
 266     {
 267         return storeType;
 268     }
 269
 270     public void setStoreType( StoreType storeType )
 271     {
 272         this.storeType = storeType;
 273     }
 274
 275     public Path getRepositoryPath()
 276     {
 277         return repositoryPath;
 278     }
 279
 280     public void setRepositoryPath( Path repositoryPath )
 281     {
 282         this.repositoryPath = repositoryPath;
 283     }
 284
 285     public void setRepositoryPath( String repositoryPath )
 286     {
 287         this.repositoryPath = Paths.get( repositoryPath );
 288         if ( !Files.exists( this.repositoryPath ) )
 289         {
 290             try
 291             {
 292                 Files.createDirectories( this.repositoryPath );
 293             }
 294             catch ( IOException e )
 295             {
 296                 log.error( e.getMessage(), e );
 297                 throw new IllegalArgumentException( "cannot create directory:" + repositoryPath, e );
 298             }
 299         }
 300     }
 301
 302     private void createExecutor()
 303     {
 304         if (executorService ==null )
 305         {
 306             executorService = Executors.newCachedThreadPool();
 307         }
 308
 309 //
 310 //        ThreadPoolExecutor executor =
 311 //            new ThreadPoolExecutor( 0, 5, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(),
 312 //                                    new ThreadFactory()
 313 //                                    {
 314 //                                        private final AtomicInteger counter = new AtomicInteger();
 315 //
 316 //                                        private final Thread.UncaughtExceptionHandler handler =
 317 //                                            new Thread.UncaughtExceptionHandler()
 318 //                                            {
 319 //                                                @Override
 320 //                                                public void uncaughtException( Thread t, Throwable e )
 321 //                                                {
 322 //                                                    log.warn( "Error occurred in asynchronous processing ", e );
 323 //                                                }
 324 //                                            };
 325 //
 326 //                                        @Override
 327 //                                        public Thread newThread( @Nonnull Runnable r )
 328 //                                        {
 329 //                                            Thread thread = new Thread( r, createName() );
 330 //                                            thread.setDaemon( true );
 331 //                                            thread.setPriority( Thread.MIN_PRIORITY );
 332 //                                            thread.setUncaughtExceptionHandler( handler );
 333 //                                            return thread;
 334 //                                        }
 335 //
 336 //                                        private String createName()
 337 //                                        {
 338 //                                            return "oak-lucene-" + counter.getAndIncrement();
 339 //                                        }
 340 //                                    } );
 341 //        executor.setKeepAliveTime( 1, TimeUnit.MINUTES );
 342 //        executor.allowCoreThreadTimeOut( true );
 343 //        return executor;
 344     }
 345
 346 }