source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.maven.scheduler.indexing;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.indexer.ArchivaIndexingContext;
  22 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  23 import org.apache.archiva.components.taskqueue.Task;
  24 import org.apache.archiva.components.taskqueue.execution.TaskExecutionException;
  25 import org.apache.archiva.components.taskqueue.execution.TaskExecutor;
  26 import org.apache.archiva.repository.ManagedRepository;
  27 import org.apache.archiva.repository.features.IndexCreationFeature;
  28 import org.apache.archiva.scheduler.indexing.ArtifactIndexingTask;
  29 import org.apache.maven.index.ArtifactContext;
  30 import org.apache.maven.index.ArtifactContextProducer;
  31 import org.apache.maven.index.DefaultScannerListener;
  32 import org.apache.maven.index.FlatSearchRequest;
  33 import org.apache.maven.index.FlatSearchResponse;
  34 import org.apache.maven.index.Indexer;
  35 import org.apache.maven.index.IndexerEngine;
  36 import org.apache.maven.index.MAVEN;
  37 import org.apache.maven.index.Scanner;
  38 import org.apache.maven.index.ScanningRequest;
  39 import org.apache.maven.index.ScanningResult;
  40 import org.apache.maven.index.context.IndexingContext;
  41 import org.apache.maven.index.expr.SourcedSearchExpression;
  42 import org.apache.maven.index.packer.IndexPacker;
  43 import org.apache.maven.index.packer.IndexPackingRequest;
  44 import org.apache.maven.index_shaded.lucene.search.BooleanClause;
  45 import org.apache.maven.index_shaded.lucene.search.BooleanQuery;
  46 import org.slf4j.Logger;
  47 import org.slf4j.LoggerFactory;
  48 import org.springframework.stereotype.Service;
  49
  50 import javax.inject.Inject;
  51 import java.io.IOException;
  52 import java.nio.file.Path;
  53
  54 /**
  55  * ArchivaIndexingTaskExecutor Executes all indexing tasks. Adding, updating and removing artifacts from the index are
  56  * all performed by this executor. Add and update artifact in index tasks are added in the indexing task queue by the
  57  * NexusIndexerConsumer while remove artifact from index tasks are added by the LuceneCleanupRemoveIndexedConsumer.
  58  */
  59 @Service( "taskExecutor#indexing" )
  60 public class ArchivaIndexingTaskExecutor
  61     implements TaskExecutor
  62 {
  63     private Logger log = LoggerFactory.getLogger( ArchivaIndexingTaskExecutor.class );
  64
  65     @Inject
  66     private IndexPacker indexPacker;
  67
  68     @Inject
  69     private ArtifactContextProducer artifactContextProducer;
  70
  71     @Inject
  72     private Indexer indexer;
  73
  74     @Inject
  75     private Scanner scanner;
  76
  77     @Inject
  78     IndexerEngine indexerEngine;
  79
  80     /**
  81      * depending on current {@link Task} you have.
  82      * If {@link org.apache.archiva.scheduler.indexing.ArtifactIndexingTask.Action#FINISH} &amp;&amp; isExecuteOnEntireRepo:
  83      * repository will be scanned.
  84      *
  85      * @param task
  86      * @throws TaskExecutionException
  87      */
  88     @Override
  89     public void executeTask( Task task )
  90         throws TaskExecutionException
  91     {
  92         ArtifactIndexingTask indexingTask = (ArtifactIndexingTask) task;
  93
  94         ManagedRepository repository = indexingTask.getRepository( );
  95         ArchivaIndexingContext archivaContext = indexingTask.getContext( );
  96         IndexingContext context = null;
  97         try
  98         {
  99             context = archivaContext.getBaseContext( IndexingContext.class );
 100         }
 101         catch ( UnsupportedBaseContextException e )
 102         {
 103             throw new TaskExecutionException( "Bad repository type.", e );
 104         }
 105
 106         if ( ArtifactIndexingTask.Action.FINISH.equals( indexingTask.getAction( ) )
 107             && indexingTask.isExecuteOnEntireRepo( ) )
 108         {
 109             long start = System.currentTimeMillis( );
 110             try
 111             {
 112                 context.updateTimestamp( );
 113                 DefaultScannerListener listener = new DefaultScannerListener( context, indexerEngine, true, null );
 114                 ScanningRequest request = new ScanningRequest( context, listener );
 115                 ScanningResult result = scanner.scan( request );
 116                 if ( result.hasExceptions( ) )
 117                 {
 118                     log.error( "Exceptions occured during index scan of " + context.getId( ) );
 119                     result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 120                         s -> log.error( "Message: " + s )
 121                     );
 122                 }
 123             }
 124             catch ( IOException e )
 125             {
 126                 log.error( "Error during context scan {}: {}", context.getId( ), context.getIndexDirectory( ) );
 127             }
 128             long end = System.currentTimeMillis( );
 129             log.info( "indexed maven repository: {}, onlyUpdate: {}, time {} ms", repository.getId( ),
 130                 indexingTask.isOnlyUpdate( ), ( end - start ) );
 131             log.debug( "Finishing indexing task on repo: {}", repository.getId( ) );
 132             finishIndexingTask( indexingTask, repository, context );
 133         }
 134         else
 135         {
 136             // create context if not a repo scan request
 137             if ( !indexingTask.isExecuteOnEntireRepo( ) )
 138             {
 139                 try
 140                 {
 141                     log.debug( "Creating indexing context on resource: {}", //
 142                         ( indexingTask.getResourceFile( ) == null
 143                             ? "none"
 144                             : indexingTask.getResourceFile( ) ) );
 145                     archivaContext = repository.getIndexingContext( );
 146                     context = archivaContext.getBaseContext( IndexingContext.class );
 147                 }
 148                 catch ( UnsupportedBaseContextException e )
 149                 {
 150                     log.error( "Error occurred while creating context: {}", e.getMessage( ) );
 151                     throw new TaskExecutionException( "Error occurred while creating context: " + e.getMessage( ), e );
 152                 }
 153             }
 154
 155             if ( context == null || context.getIndexDirectory( ) == null )
 156             {
 157                 throw new TaskExecutionException( "Trying to index an artifact but the context is already closed" );
 158             }
 159
 160             try
 161             {
 162                 Path artifactFile = indexingTask.getResourceFile( );
 163                 if ( artifactFile == null )
 164                 {
 165                     log.debug( "no artifact pass in indexing task so skip it" );
 166                 }
 167                 else
 168                 {
 169                     ArtifactContext ac = artifactContextProducer.getArtifactContext( context, artifactFile.toFile( ) );
 170
 171                     if ( ac != null )
 172                     {
 173                         // MRM-1779 pom must be indexed too
 174                         // TODO make that configurable?
 175                         if ( artifactFile.getFileName( ).toString( ).endsWith( ".pom" ) )
 176                         {
 177                             ac.getArtifactInfo( ).setFileExtension( "pom" );
 178                             ac.getArtifactInfo( ).setPackaging( "pom" );
 179                             ac.getArtifactInfo( ).setClassifier( "pom" );
 180                         }
 181                         if ( indexingTask.getAction( ).equals( ArtifactIndexingTask.Action.ADD ) )
 182                         {
 183                             //IndexSearcher s = context.getIndexSearcher();
 184                             //String uinfo = ac.getArtifactInfo().getUinfo();
 185                             //TopDocs d = s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), 1 );
 186
 187                             BooleanQuery.Builder qb = new BooleanQuery.Builder();
 188                             qb.add( indexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression(
 189                                 ac.getArtifactInfo( ).getGroupId( ) ) ), BooleanClause.Occur.MUST );
 190                             qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new SourcedSearchExpression(
 191                                 ac.getArtifactInfo( ).getArtifactId( ) ) ), BooleanClause.Occur.MUST );
 192                             qb.add( indexer.constructQuery( MAVEN.VERSION, new SourcedSearchExpression(
 193                                 ac.getArtifactInfo( ).getVersion( ) ) ), BooleanClause.Occur.MUST );
 194                             if ( ac.getArtifactInfo( ).getClassifier( ) != null )
 195                             {
 196                                 qb.add( indexer.constructQuery( MAVEN.CLASSIFIER, new SourcedSearchExpression(
 197                                     ac.getArtifactInfo( ).getClassifier( ) ) ), BooleanClause.Occur.MUST );
 198                             }
 199                             if ( ac.getArtifactInfo( ).getPackaging( ) != null )
 200                             {
 201                                 qb.add( indexer.constructQuery( MAVEN.PACKAGING, new SourcedSearchExpression(
 202                                     ac.getArtifactInfo( ).getPackaging( ) ) ), BooleanClause.Occur.MUST );
 203                             }
 204                             FlatSearchRequest flatSearchRequest = new FlatSearchRequest( qb.build(), context );
 205                             FlatSearchResponse flatSearchResponse = indexer.searchFlat( flatSearchRequest );
 206                             if ( flatSearchResponse.getResults( ).isEmpty( ) )
 207                             {
 208                                 log.debug( "Adding artifact '{}' to index..", ac.getArtifactInfo( ) );
 209                                 indexerEngine.index( context, ac );
 210                             }
 211                             else
 212                             {
 213                                 log.debug( "Updating artifact '{}' in index..", ac.getArtifactInfo( ) );
 214                                 // TODO check if update exists !!
 215                                 indexerEngine.update( context, ac );
 216                             }
 217
 218                             context.updateTimestamp( );
 219                             context.commit( );
 220
 221
 222                         }
 223                         else
 224                         {
 225                             log.debug( "Removing artifact '{}' from index..", ac.getArtifactInfo( ) );
 226                             indexerEngine.remove( context, ac );
 227                         }
 228                     }
 229                 }
 230                 // close the context if not a repo scan request
 231                 if ( !indexingTask.isExecuteOnEntireRepo( ) )
 232                 {
 233                     log.debug( "Finishing indexing task on resource file : {}", indexingTask.getResourceFile( ) != null
 234                         ? indexingTask.getResourceFile( )
 235                         : " none " );
 236                     finishIndexingTask( indexingTask, repository, context );
 237                 }
 238             }
 239             catch ( IOException e )
 240             {
 241                 log.error( "Error occurred while executing indexing task '{}': {}", indexingTask, e.getMessage( ),
 242                     e );
 243                 throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'",
 244                     e );
 245             }
 246         }
 247
 248     }
 249
 250     private void finishIndexingTask( ArtifactIndexingTask indexingTask, ManagedRepository repository,
 251                                      IndexingContext context )
 252         throws TaskExecutionException
 253     {
 254         try
 255         {
 256
 257             log.debug( "Finishing indexing" );
 258             context.optimize( );
 259
 260             if ( repository.supportsFeature( IndexCreationFeature.class ) )
 261             {
 262                 IndexCreationFeature icf = repository.getFeature( IndexCreationFeature.class ).get( );
 263                 if ( !icf.isSkipPackedIndexCreation( ) && icf.getLocalPackedIndexPath( ) != null && icf.getLocalIndexPath().getFilePath()!=null )
 264                 {
 265
 266                     log.debug( "Creating packed index from {} on {}", context.getIndexDirectoryFile( ), icf.getLocalPackedIndexPath( ) );
 267                     IndexPackingRequest request = new IndexPackingRequest( context, //
 268                         context.acquireIndexSearcher( ).getIndexReader( ),
 269                         //
 270                         icf.getLocalPackedIndexPath( ).getFilePath().toFile( ) );
 271
 272                     indexPacker.packIndex( request );
 273                     context.updateTimestamp( true );
 274
 275                     log.debug( "Index file packed at '{}'.", icf.getLocalPackedIndexPath( ) );
 276                 }
 277                 else
 278                 {
 279                     log.debug( "skip packed index creation" );
 280                 }
 281             }
 282             else
 283             {
 284                 log.debug( "skip packed index creation" );
 285             }
 286         }
 287         catch ( IOException e )
 288         {
 289             log.error( "Error occurred while executing indexing task '{}': {}", indexingTask, e.getMessage( ) );
 290             throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'",
 291                 e );
 292         }
 293     }
 294
 295     public void setIndexPacker( IndexPacker indexPacker )
 296     {
 297         this.indexPacker = indexPacker;
 298     }
 299
 300 }