123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300 |
- package org.apache.archiva.maven.scheduler.indexing;
-
- /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
- import org.apache.archiva.indexer.ArchivaIndexingContext;
- import org.apache.archiva.indexer.UnsupportedBaseContextException;
- import org.apache.archiva.components.taskqueue.Task;
- import org.apache.archiva.components.taskqueue.execution.TaskExecutionException;
- import org.apache.archiva.components.taskqueue.execution.TaskExecutor;
- import org.apache.archiva.repository.ManagedRepository;
- import org.apache.archiva.repository.features.IndexCreationFeature;
- import org.apache.archiva.scheduler.indexing.ArtifactIndexingTask;
- import org.apache.maven.index.ArtifactContext;
- import org.apache.maven.index.ArtifactContextProducer;
- import org.apache.maven.index.DefaultScannerListener;
- import org.apache.maven.index.FlatSearchRequest;
- import org.apache.maven.index.FlatSearchResponse;
- import org.apache.maven.index.Indexer;
- import org.apache.maven.index.IndexerEngine;
- import org.apache.maven.index.MAVEN;
- import org.apache.maven.index.Scanner;
- import org.apache.maven.index.ScanningRequest;
- import org.apache.maven.index.ScanningResult;
- import org.apache.maven.index.context.IndexingContext;
- import org.apache.maven.index.expr.SourcedSearchExpression;
- import org.apache.maven.index.packer.IndexPacker;
- import org.apache.maven.index.packer.IndexPackingRequest;
- import org.apache.maven.index_shaded.lucene.search.BooleanClause;
- import org.apache.maven.index_shaded.lucene.search.BooleanQuery;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import org.springframework.stereotype.Service;
-
- import javax.inject.Inject;
- import java.io.IOException;
- import java.nio.file.Path;
-
- /**
- * ArchivaIndexingTaskExecutor Executes all indexing tasks. Adding, updating and removing artifacts from the index are
- * all performed by this executor. Add and update artifact in index tasks are added in the indexing task queue by the
- * NexusIndexerConsumer while remove artifact from index tasks are added by the LuceneCleanupRemoveIndexedConsumer.
- */
- @Service( "taskExecutor#indexing" )
- public class ArchivaIndexingTaskExecutor
- implements TaskExecutor
- {
- private Logger log = LoggerFactory.getLogger( ArchivaIndexingTaskExecutor.class );
-
- @Inject
- private IndexPacker indexPacker;
-
- @Inject
- private ArtifactContextProducer artifactContextProducer;
-
- @Inject
- private Indexer indexer;
-
- @Inject
- private Scanner scanner;
-
- @Inject
- IndexerEngine indexerEngine;
-
- /**
- * depending on current {@link Task} you have.
- * If {@link org.apache.archiva.scheduler.indexing.ArtifactIndexingTask.Action#FINISH} && isExecuteOnEntireRepo:
- * repository will be scanned.
- *
- * @param task
- * @throws TaskExecutionException
- */
- @Override
- public void executeTask( Task task )
- throws TaskExecutionException
- {
- ArtifactIndexingTask indexingTask = (ArtifactIndexingTask) task;
-
- ManagedRepository repository = indexingTask.getRepository( );
- ArchivaIndexingContext archivaContext = indexingTask.getContext( );
- IndexingContext context = null;
- try
- {
- context = archivaContext.getBaseContext( IndexingContext.class );
- }
- catch ( UnsupportedBaseContextException e )
- {
- throw new TaskExecutionException( "Bad repository type.", e );
- }
-
- if ( ArtifactIndexingTask.Action.FINISH.equals( indexingTask.getAction( ) )
- && indexingTask.isExecuteOnEntireRepo( ) )
- {
- long start = System.currentTimeMillis( );
- try
- {
- context.updateTimestamp( );
- DefaultScannerListener listener = new DefaultScannerListener( context, indexerEngine, true, null );
- ScanningRequest request = new ScanningRequest( context, listener );
- ScanningResult result = scanner.scan( request );
- if ( result.hasExceptions( ) )
- {
- log.error( "Exceptions occured during index scan of " + context.getId( ) );
- result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
- s -> log.error( "Message: " + s )
- );
- }
- }
- catch ( IOException e )
- {
- log.error( "Error during context scan {}: {}", context.getId( ), context.getIndexDirectory( ) );
- }
- long end = System.currentTimeMillis( );
- log.info( "indexed maven repository: {}, onlyUpdate: {}, time {} ms", repository.getId( ),
- indexingTask.isOnlyUpdate( ), ( end - start ) );
- log.debug( "Finishing indexing task on repo: {}", repository.getId( ) );
- finishIndexingTask( indexingTask, repository, context );
- }
- else
- {
- // create context if not a repo scan request
- if ( !indexingTask.isExecuteOnEntireRepo( ) )
- {
- try
- {
- log.debug( "Creating indexing context on resource: {}", //
- ( indexingTask.getResourceFile( ) == null
- ? "none"
- : indexingTask.getResourceFile( ) ) );
- archivaContext = repository.getIndexingContext( );
- context = archivaContext.getBaseContext( IndexingContext.class );
- }
- catch ( UnsupportedBaseContextException e )
- {
- log.error( "Error occurred while creating context: {}", e.getMessage( ) );
- throw new TaskExecutionException( "Error occurred while creating context: " + e.getMessage( ), e );
- }
- }
-
- if ( context == null || context.getIndexDirectory( ) == null )
- {
- throw new TaskExecutionException( "Trying to index an artifact but the context is already closed" );
- }
-
- try
- {
- Path artifactFile = indexingTask.getResourceFile( );
- if ( artifactFile == null )
- {
- log.debug( "no artifact pass in indexing task so skip it" );
- }
- else
- {
- ArtifactContext ac = artifactContextProducer.getArtifactContext( context, artifactFile.toFile( ) );
-
- if ( ac != null )
- {
- // MRM-1779 pom must be indexed too
- // TODO make that configurable?
- if ( artifactFile.getFileName( ).toString( ).endsWith( ".pom" ) )
- {
- ac.getArtifactInfo( ).setFileExtension( "pom" );
- ac.getArtifactInfo( ).setPackaging( "pom" );
- ac.getArtifactInfo( ).setClassifier( "pom" );
- }
- if ( indexingTask.getAction( ).equals( ArtifactIndexingTask.Action.ADD ) )
- {
- //IndexSearcher s = context.getIndexSearcher();
- //String uinfo = ac.getArtifactInfo().getUinfo();
- //TopDocs d = s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), 1 );
-
- BooleanQuery.Builder qb = new BooleanQuery.Builder();
- qb.add( indexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression(
- ac.getArtifactInfo( ).getGroupId( ) ) ), BooleanClause.Occur.MUST );
- qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new SourcedSearchExpression(
- ac.getArtifactInfo( ).getArtifactId( ) ) ), BooleanClause.Occur.MUST );
- qb.add( indexer.constructQuery( MAVEN.VERSION, new SourcedSearchExpression(
- ac.getArtifactInfo( ).getVersion( ) ) ), BooleanClause.Occur.MUST );
- if ( ac.getArtifactInfo( ).getClassifier( ) != null )
- {
- qb.add( indexer.constructQuery( MAVEN.CLASSIFIER, new SourcedSearchExpression(
- ac.getArtifactInfo( ).getClassifier( ) ) ), BooleanClause.Occur.MUST );
- }
- if ( ac.getArtifactInfo( ).getPackaging( ) != null )
- {
- qb.add( indexer.constructQuery( MAVEN.PACKAGING, new SourcedSearchExpression(
- ac.getArtifactInfo( ).getPackaging( ) ) ), BooleanClause.Occur.MUST );
- }
- FlatSearchRequest flatSearchRequest = new FlatSearchRequest( qb.build(), context );
- FlatSearchResponse flatSearchResponse = indexer.searchFlat( flatSearchRequest );
- if ( flatSearchResponse.getResults( ).isEmpty( ) )
- {
- log.debug( "Adding artifact '{}' to index..", ac.getArtifactInfo( ) );
- indexerEngine.index( context, ac );
- }
- else
- {
- log.debug( "Updating artifact '{}' in index..", ac.getArtifactInfo( ) );
- // TODO check if update exists !!
- indexerEngine.update( context, ac );
- }
-
- context.updateTimestamp( );
- context.commit( );
-
-
- }
- else
- {
- log.debug( "Removing artifact '{}' from index..", ac.getArtifactInfo( ) );
- indexerEngine.remove( context, ac );
- }
- }
- }
- // close the context if not a repo scan request
- if ( !indexingTask.isExecuteOnEntireRepo( ) )
- {
- log.debug( "Finishing indexing task on resource file : {}", indexingTask.getResourceFile( ) != null
- ? indexingTask.getResourceFile( )
- : " none " );
- finishIndexingTask( indexingTask, repository, context );
- }
- }
- catch ( IOException e )
- {
- log.error( "Error occurred while executing indexing task '{}': {}", indexingTask, e.getMessage( ),
- e );
- throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'",
- e );
- }
- }
-
- }
-
- private void finishIndexingTask( ArtifactIndexingTask indexingTask, ManagedRepository repository,
- IndexingContext context )
- throws TaskExecutionException
- {
- try
- {
-
- log.debug( "Finishing indexing" );
- context.optimize( );
-
- if ( repository.supportsFeature( IndexCreationFeature.class ) )
- {
- IndexCreationFeature icf = repository.getFeature( IndexCreationFeature.class );
- if ( !icf.isSkipPackedIndexCreation( ) && icf.getLocalPackedIndexPath( ) != null && icf.getLocalIndexPath().getFilePath()!=null )
- {
-
- log.debug( "Creating packed index from {} on {}", context.getIndexDirectoryFile( ), icf.getLocalPackedIndexPath( ) );
- IndexPackingRequest request = new IndexPackingRequest( context, //
- context.acquireIndexSearcher( ).getIndexReader( ),
- //
- icf.getLocalPackedIndexPath( ).getFilePath().toFile( ) );
-
- indexPacker.packIndex( request );
- context.updateTimestamp( true );
-
- log.debug( "Index file packed at '{}'.", icf.getLocalPackedIndexPath( ) );
- }
- else
- {
- log.debug( "skip packed index creation" );
- }
- }
- else
- {
- log.debug( "skip packed index creation" );
- }
- }
- catch ( IOException e )
- {
- log.error( "Error occurred while executing indexing task '{}': {}", indexingTask, e.getMessage( ) );
- throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'",
- e );
- }
- }
-
- public void setIndexPacker( IndexPacker indexPacker )
- {
- this.indexPacker = indexPacker;
- }
-
- }
|