source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing.maven;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the Li
  16  * cense is distributed on an
  17  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  18  * KIND, either express or implied.  See the License for the
  19  * specific language governing permissions and limitations
  20  * under the License.
  21  */
  22
  23 import org.apache.archiva.indexer.ArchivaIndexingContext;
  24 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  25 import org.apache.archiva.redback.components.taskqueue.Task;
  26 import org.apache.archiva.redback.components.taskqueue.execution.TaskExecutionException;
  27 import org.apache.archiva.redback.components.taskqueue.execution.TaskExecutor;
  28 import org.apache.archiva.repository.ManagedRepository;
  29 import org.apache.archiva.repository.features.IndexCreationFeature;
  30 import org.apache.archiva.scheduler.indexing.ArtifactIndexingTask;
  31 import org.apache.maven.index.ArtifactContext;
  32 import org.apache.maven.index.ArtifactContextProducer;
  33 import org.apache.maven.index.DefaultScannerListener;
  34 import org.apache.maven.index.FlatSearchRequest;
  35 import org.apache.maven.index.FlatSearchResponse;
  36 import org.apache.maven.index.Indexer;
  37 import org.apache.maven.index.IndexerEngine;
  38 import org.apache.maven.index.MAVEN;
  39 import org.apache.maven.index.Scanner;
  40 import org.apache.maven.index.ScanningRequest;
  41 import org.apache.maven.index.ScanningResult;
  42 import org.apache.maven.index.context.IndexingContext;
  43 import org.apache.maven.index.expr.SourcedSearchExpression;
  44 import org.apache.maven.index.packer.IndexPacker;
  45 import org.apache.maven.index.packer.IndexPackingRequest;
  46 import org.apache.maven.index_shaded.lucene.search.BooleanClause;
  47 import org.apache.maven.index_shaded.lucene.search.BooleanQuery;
  48 import org.slf4j.Logger;
  49 import org.slf4j.LoggerFactory;
  50 import org.springframework.stereotype.Service;
  51
  52 import javax.inject.Inject;
  53 import java.io.IOException;
  54 import java.nio.file.Path;
  55
  56 /**
  57  * ArchivaIndexingTaskExecutor Executes all indexing tasks. Adding, updating and removing artifacts from the index are
  58  * all performed by this executor. Add and update artifact in index tasks are added in the indexing task queue by the
  59  * NexusIndexerConsumer while remove artifact from index tasks are added by the LuceneCleanupRemoveIndexedConsumer.
  60  */
  61 @Service( "taskExecutor#indexing" )
  62 public class ArchivaIndexingTaskExecutor
  63     implements TaskExecutor
  64 {
  65     private Logger log = LoggerFactory.getLogger( ArchivaIndexingTaskExecutor.class );
  66
  67     @Inject
  68     private IndexPacker indexPacker;
  69
  70     @Inject
  71     private ArtifactContextProducer artifactContextProducer;
  72
  73     @Inject
  74     private Indexer indexer;
  75
  76     @Inject
  77     private Scanner scanner;
  78
  79     @Inject
  80     IndexerEngine indexerEngine;
  81
  82     /**
  83      * depending on current {@link Task} you have.
  84      * If {@link org.apache.archiva.scheduler.indexing.ArtifactIndexingTask.Action#FINISH} &amp;&amp; isExecuteOnEntireRepo:
  85      * repository will be scanned.
  86      *
  87      * @param task
  88      * @throws TaskExecutionException
  89      */
  90     @Override
  91     public void executeTask( Task task )
  92         throws TaskExecutionException
  93     {
  94         ArtifactIndexingTask indexingTask = (ArtifactIndexingTask) task;
  95
  96         ManagedRepository repository = indexingTask.getRepository( );
  97         ArchivaIndexingContext archivaContext = indexingTask.getContext( );
  98         IndexingContext context = null;
  99         try
 100         {
 101             context = archivaContext.getBaseContext( IndexingContext.class );
 102         }
 103         catch ( UnsupportedBaseContextException e )
 104         {
 105             throw new TaskExecutionException( "Bad repository type.", e );
 106         }
 107
 108         if ( ArtifactIndexingTask.Action.FINISH.equals( indexingTask.getAction( ) )
 109             && indexingTask.isExecuteOnEntireRepo( ) )
 110         {
 111             long start = System.currentTimeMillis( );
 112             try
 113             {
 114                 context.updateTimestamp( );
 115                 DefaultScannerListener listener = new DefaultScannerListener( context, indexerEngine, true, null );
 116                 ScanningRequest request = new ScanningRequest( context, listener );
 117                 ScanningResult result = scanner.scan( request );
 118                 if ( result.hasExceptions( ) )
 119                 {
 120                     log.error( "Exceptions occured during index scan of " + context.getId( ) );
 121                     result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 122                         s -> log.error( "Message: " + s )
 123                     );
 124                 }
 125             }
 126             catch ( IOException e )
 127             {
 128                 log.error( "Error during context scan {}: {}", context.getId( ), context.getIndexDirectory( ) );
 129             }
 130             long end = System.currentTimeMillis( );
 131             log.info( "indexed maven repository: {}, onlyUpdate: {}, time {} ms", repository.getId( ),
 132                 indexingTask.isOnlyUpdate( ), ( end - start ) );
 133             log.debug( "Finishing indexing task on repo: {}", repository.getId( ) );
 134             finishIndexingTask( indexingTask, repository, context );
 135         }
 136         else
 137         {
 138             // create context if not a repo scan request
 139             if ( !indexingTask.isExecuteOnEntireRepo( ) )
 140             {
 141                 try
 142                 {
 143                     log.debug( "Creating indexing context on resource: {}", //
 144                         ( indexingTask.getResourceFile( ) == null
 145                             ? "none"
 146                             : indexingTask.getResourceFile( ) ) );
 147                     archivaContext = repository.getIndexingContext( );
 148                     context = archivaContext.getBaseContext( IndexingContext.class );
 149                 }
 150                 catch ( UnsupportedBaseContextException e )
 151                 {
 152                     log.error( "Error occurred while creating context: {}", e.getMessage( ) );
 153                     throw new TaskExecutionException( "Error occurred while creating context: " + e.getMessage( ), e );
 154                 }
 155             }
 156
 157             if ( context == null || context.getIndexDirectory( ) == null )
 158             {
 159                 throw new TaskExecutionException( "Trying to index an artifact but the context is already closed" );
 160             }
 161
 162             try
 163             {
 164                 Path artifactFile = indexingTask.getResourceFile( );
 165                 if ( artifactFile == null )
 166                 {
 167                     log.debug( "no artifact pass in indexing task so skip it" );
 168                 }
 169                 else
 170                 {
 171                     ArtifactContext ac = artifactContextProducer.getArtifactContext( context, artifactFile.toFile( ) );
 172
 173                     if ( ac != null )
 174                     {
 175                         // MRM-1779 pom must be indexed too
 176                         // TODO make that configurable?
 177                         if ( artifactFile.getFileName( ).toString( ).endsWith( ".pom" ) )
 178                         {
 179                             ac.getArtifactInfo( ).setFileExtension( "pom" );
 180                             ac.getArtifactInfo( ).setPackaging( "pom" );
 181                             ac.getArtifactInfo( ).setClassifier( "pom" );
 182                         }
 183                         if ( indexingTask.getAction( ).equals( ArtifactIndexingTask.Action.ADD ) )
 184                         {
 185                             //IndexSearcher s = context.getIndexSearcher();
 186                             //String uinfo = ac.getArtifactInfo().getUinfo();
 187                             //TopDocs d = s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), 1 );
 188
 189                             BooleanQuery.Builder qb = new BooleanQuery.Builder();
 190                             qb.add( indexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression(
 191                                 ac.getArtifactInfo( ).getGroupId( ) ) ), BooleanClause.Occur.MUST );
 192                             qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new SourcedSearchExpression(
 193                                 ac.getArtifactInfo( ).getArtifactId( ) ) ), BooleanClause.Occur.MUST );
 194                             qb.add( indexer.constructQuery( MAVEN.VERSION, new SourcedSearchExpression(
 195                                 ac.getArtifactInfo( ).getVersion( ) ) ), BooleanClause.Occur.MUST );
 196                             if ( ac.getArtifactInfo( ).getClassifier( ) != null )
 197                             {
 198                                 qb.add( indexer.constructQuery( MAVEN.CLASSIFIER, new SourcedSearchExpression(
 199                                     ac.getArtifactInfo( ).getClassifier( ) ) ), BooleanClause.Occur.MUST );
 200                             }
 201                             if ( ac.getArtifactInfo( ).getPackaging( ) != null )
 202                             {
 203                                 qb.add( indexer.constructQuery( MAVEN.PACKAGING, new SourcedSearchExpression(
 204                                     ac.getArtifactInfo( ).getPackaging( ) ) ), BooleanClause.Occur.MUST );
 205                             }
 206                             FlatSearchRequest flatSearchRequest = new FlatSearchRequest( qb.build(), context );
 207                             FlatSearchResponse flatSearchResponse = indexer.searchFlat( flatSearchRequest );
 208                             if ( flatSearchResponse.getResults( ).isEmpty( ) )
 209                             {
 210                                 log.debug( "Adding artifact '{}' to index..", ac.getArtifactInfo( ) );
 211                                 indexerEngine.index( context, ac );
 212                             }
 213                             else
 214                             {
 215                                 log.debug( "Updating artifact '{}' in index..", ac.getArtifactInfo( ) );
 216                                 // TODO check if update exists !!
 217                                 indexerEngine.update( context, ac );
 218                             }
 219
 220                             context.updateTimestamp( );
 221                             context.commit( );
 222
 223
 224                         }
 225                         else
 226                         {
 227                             log.debug( "Removing artifact '{}' from index..", ac.getArtifactInfo( ) );
 228                             indexerEngine.remove( context, ac );
 229                         }
 230                     }
 231                 }
 232                 // close the context if not a repo scan request
 233                 if ( !indexingTask.isExecuteOnEntireRepo( ) )
 234                 {
 235                     log.debug( "Finishing indexing task on resource file : {}", indexingTask.getResourceFile( ) != null
 236                         ? indexingTask.getResourceFile( )
 237                         : " none " );
 238                     finishIndexingTask( indexingTask, repository, context );
 239                 }
 240             }
 241             catch ( IOException e )
 242             {
 243                 log.error( "Error occurred while executing indexing task '{}': {}", indexingTask, e.getMessage( ),
 244                     e );
 245                 throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'",
 246                     e );
 247             }
 248         }
 249
 250     }
 251
 252     private void finishIndexingTask( ArtifactIndexingTask indexingTask, ManagedRepository repository,
 253                                      IndexingContext context )
 254         throws TaskExecutionException
 255     {
 256         try
 257         {
 258
 259             log.debug( "Finishing indexing" );
 260             context.optimize( );
 261
 262             if ( repository.supportsFeature( IndexCreationFeature.class ) )
 263             {
 264                 IndexCreationFeature icf = repository.getFeature( IndexCreationFeature.class ).get( );
 265                 if ( !icf.isSkipPackedIndexCreation( ) && icf.getLocalPackedIndexPath( ) != null && icf.getLocalIndexPath().getFilePath()!=null )
 266                 {
 267
 268                     log.debug( "Creating packed index from {} on {}", context.getIndexDirectoryFile( ), icf.getLocalPackedIndexPath( ) );
 269                     IndexPackingRequest request = new IndexPackingRequest( context, //
 270                         context.acquireIndexSearcher( ).getIndexReader( ),
 271                         //
 272                         icf.getLocalPackedIndexPath( ).getFilePath().toFile( ) );
 273
 274                     indexPacker.packIndex( request );
 275                     context.updateTimestamp( true );
 276
 277                     log.debug( "Index file packed at '{}'.", icf.getLocalPackedIndexPath( ) );
 278                 }
 279                 else
 280                 {
 281                     log.debug( "skip packed index creation" );
 282                 }
 283             }
 284             else
 285             {
 286                 log.debug( "skip packed index creation" );
 287             }
 288         }
 289         catch ( IOException e )
 290         {
 291             log.error( "Error occurred while executing indexing task '{}': {}", indexingTask, e.getMessage( ) );
 292             throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'",
 293                 e );
 294         }
 295     }
 296
 297     public void setIndexPacker( IndexPacker indexPacker )
 298     {
 299         this.indexPacker = indexPacker;
 300     }
 301
 302 }