diff options
author | Brett Porter <brett@apache.org> | 2009-10-14 15:15:43 +0000 |
---|---|---|
committer | Brett Porter <brett@apache.org> | 2009-10-14 15:15:43 +0000 |
commit | c30389bfe8feecf769293208990cfb3daba69c75 (patch) | |
tree | 8919263b69e30ac12f45cdcd8dc927814899196f | |
parent | 0cf3f93e5aa693ca14bbe52387f4cc5b5be92b06 (diff) | |
download | archiva-c30389bfe8feecf769293208990cfb3daba69c75.tar.gz archiva-c30389bfe8feecf769293208990cfb3daba69c75.zip |
[MRM-1264] Only create the DefaultIndexingContext once for a scan to reduce the amount of garbage collection. Also move optimization to end of scan to increase performance.
git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@825163 13f79535-47bb-0310-9956-ffa450edef68
9 files changed, 294 insertions, 253 deletions
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumer.java index ef310b04f..8e57ff93b 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumer.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumer.java @@ -20,6 +20,7 @@ package org.apache.archiva.consumers.lucene; */ import java.io.File; +import java.io.IOException; import java.util.List; import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; @@ -36,11 +37,12 @@ import org.apache.maven.archiva.scheduled.tasks.TaskCreator; import org.codehaus.plexus.taskqueue.TaskQueueException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonatype.nexus.index.context.IndexingContext; +import org.sonatype.nexus.index.context.UnsupportedExistingLuceneIndexException; /** - * LuceneCleanupRemoveIndexedConsumer - * - * Clean up the index of artifacts that are no longer existing in the file system (managed repositories). + * LuceneCleanupRemoveIndexedConsumer Clean up the index of artifacts that are no longer existing in the file system + * (managed repositories). * * @version $Id$ */ @@ -51,15 +53,15 @@ public class LuceneCleanupRemoveIndexedConsumer private static final Logger log = LoggerFactory.getLogger( LuceneCleanupRemoveIndexedConsumer.class ); private RepositoryContentFactory repoFactory; - + private ArchivaTaskScheduler scheduler; - + public LuceneCleanupRemoveIndexedConsumer( RepositoryContentFactory repoFactory, ArchivaTaskScheduler scheduler ) { this.repoFactory = repoFactory; - this.scheduler = scheduler; + this.scheduler = scheduler; } - + public void beginScan() { } @@ -69,7 +71,7 @@ public class LuceneCleanupRemoveIndexedConsumer } public List<String> getIncludedTypes() - { + { return null; } @@ -77,11 +79,10 @@ public class LuceneCleanupRemoveIndexedConsumer throws ConsumerException { ManagedRepositoryContent repoContent = null; - + try { - repoContent = - repoFactory.getManagedRepositoryContent( artifact.getModel().getRepositoryId() ); + repoContent = repoFactory.getManagedRepositoryContent( artifact.getModel().getRepositoryId() ); } catch ( RepositoryException e ) { @@ -89,32 +90,59 @@ public class LuceneCleanupRemoveIndexedConsumer } ManagedRepositoryConfiguration repository = repoContent.getRepository(); - + + IndexingContext context = null; try { File artifactFile = new File( repoContent.getRepoRoot(), repoContent.toPath( artifact ) ); - + if ( !artifactFile.exists() ) { - ArtifactIndexingTask task = - TaskCreator.createIndexingTask( repository.getId(), artifactFile, ArtifactIndexingTask.Action.DELETE ); - + context = TaskCreator.createContext( repository ); + + ArtifactIndexingTask task = + TaskCreator.createIndexingTask( repository, artifactFile, ArtifactIndexingTask.Action.DELETE, + context ); + log.debug( "Queueing indexing task '" + task + "' to remove the artifact from the index." ); scheduler.queueIndexingTask( task ); // note we finish immediately here since it isn't done repo-by-repo. It might be nice to ensure that is // the case for optimisation though task = - TaskCreator.createIndexingTask( repository.getId(), artifactFile, ArtifactIndexingTask.Action.FINISH ); + TaskCreator.createIndexingTask( repository, artifactFile, ArtifactIndexingTask.Action.FINISH, + context ); log.debug( "Queueing indexing task + '" + task + "' to finish indexing." ); scheduler.queueIndexingTask( task ); } - - } + + } catch ( TaskQueueException e ) { throw new ConsumerException( e.getMessage() ); } + catch ( IOException e ) + { + throw new ConsumerException( e.getMessage(), e ); + } + catch ( UnsupportedExistingLuceneIndexException e ) + { + throw new ConsumerException( e.getMessage(), e ); + } + finally + { + if ( context != null ) + { + try + { + context.close( false ); + } + catch ( IOException e ) + { + log.error( e.getMessage() ); + } + } + } } public String getDescription() diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java index 94a22996f..00378c706 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java @@ -20,6 +20,7 @@ package org.apache.archiva.consumers.lucene; */ import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -33,10 +34,11 @@ import org.apache.maven.archiva.repository.content.ManagedDefaultRepositoryConte import org.apache.maven.archiva.scheduled.ArchivaTaskScheduler; import org.apache.maven.archiva.scheduled.tasks.ArtifactIndexingTask; import org.apache.maven.archiva.scheduled.tasks.TaskCreator; -import org.apache.maven.archiva.scheduled.tasks.ArtifactIndexingTask.Action; import org.codehaus.plexus.taskqueue.TaskQueueException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonatype.nexus.index.context.IndexingContext; +import org.sonatype.nexus.index.context.UnsupportedExistingLuceneIndexException; /** * Consumer for indexing the repository to provide search and IDE integration features. @@ -50,14 +52,16 @@ public class NexusIndexerConsumer private ManagedDefaultRepositoryContent repositoryContent; private File managedRepository; - + private ArchivaTaskScheduler scheduler; - + + private IndexingContext context; + public NexusIndexerConsumer( ArchivaTaskScheduler scheduler ) { this.scheduler = scheduler; } - + public String getDescription() { return "Indexes the repository to provide search and IDE integration features"; @@ -75,23 +79,34 @@ public class NexusIndexerConsumer public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered ) throws ConsumerException - { + { managedRepository = new File( repository.getLocation() ); repositoryContent = new ManagedDefaultRepositoryContent(); repositoryContent.setRepository( repository ); - - // TODO: investigate whether it is reasonable to create the indexing context here rather than file-by-file - // we may want to be able to "flush" it after every file without closing it though, if necessary + + try + { + context = TaskCreator.createContext( repository ); + } + catch ( IOException e ) + { + throw new ConsumerException( e.getMessage(), e ); + } + catch ( UnsupportedExistingLuceneIndexException e ) + { + throw new ConsumerException( e.getMessage(), e ); + } } - + public void processFile( String path ) throws ConsumerException { File artifactFile = new File( managedRepository, path ); - + ArtifactIndexingTask task = - TaskCreator.createIndexingTask( repositoryContent.getId(), artifactFile, ArtifactIndexingTask.Action.ADD ); + TaskCreator.createIndexingTask( repositoryContent.getRepository(), artifactFile, + ArtifactIndexingTask.Action.ADD, context ); try { log.debug( "Queueing indexing task + '" + task + "' to add or update the artifact in the index." ); @@ -100,13 +115,14 @@ public class NexusIndexerConsumer catch ( TaskQueueException e ) { throw new ConsumerException( e.getMessage(), e ); - } + } } public void completeScan() { ArtifactIndexingTask task = - TaskCreator.createIndexingTask( repositoryContent.getId(), null, ArtifactIndexingTask.Action.FINISH ); + TaskCreator.createIndexingTask( repositoryContent.getRepository(), null, + ArtifactIndexingTask.Action.FINISH, context ); try { log.debug( "Queueing indexing task + '" + task + "' to finish indexing." ); @@ -115,7 +131,7 @@ public class NexusIndexerConsumer catch ( TaskQueueException e ) { log.error( "Error queueing task: " + task + ": " + e.getMessage(), e ); - } + } } public List<String> getExcludes() diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumerTest.java b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumerTest.java index 4f005dff2..1eb7237f9 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumerTest.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumerTest.java @@ -49,7 +49,7 @@ public class LuceneCleanupRemoveIndexedConsumerTest private MockControl repoFactoryControl; private ManagedRepositoryConfiguration repositoryConfig; - + private TaskQueue indexingQueue; public void setUp() @@ -60,10 +60,10 @@ public class LuceneCleanupRemoveIndexedConsumerTest repoFactoryControl = MockClassControl.createControl( RepositoryContentFactory.class ); repoFactory = (RepositoryContentFactory) repoFactoryControl.getMock(); - ArchivaTaskScheduler scheduler = ( ArchivaTaskScheduler ) lookup( ArchivaTaskScheduler.class ); - + ArchivaTaskScheduler scheduler = (ArchivaTaskScheduler) lookup( ArchivaTaskScheduler.class ); + indexingQueue = (TaskQueue) lookup( TaskQueue.ROLE, "indexing" ); - + consumer = new LuceneCleanupRemoveIndexedConsumer( repoFactory, scheduler ); repositoryConfig = new ManagedRepositoryConfiguration(); @@ -93,28 +93,28 @@ public class LuceneCleanupRemoveIndexedConsumerTest ArchivaArtifact artifact = new ArchivaArtifact( "org.apache.archiva", "archiva-lucene-consumers", "1.2", null, "jar", "test-repo" ); - + ManagedRepositoryContent repoContent = new ManagedDefaultRepositoryContent(); repoContent.setRepository( repositoryConfig ); - + File artifactFile = new File( repoContent.getRepoRoot(), repoContent.toPath( artifact ) ); - + repoFactoryControl.expectAndReturn( repoFactory.getManagedRepositoryContent( repositoryConfig.getId() ), repoContent ); - - repoFactoryControl.replay(); + + repoFactoryControl.replay(); consumer.processArchivaArtifact( artifact ); - repoFactoryControl.verify(); - + repoFactoryControl.verify(); + List<ArtifactIndexingTask> queue = indexingQueue.getQueueSnapshot(); assertEquals( 2, queue.size() ); ArtifactIndexingTask task = - TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.DELETE ); + TaskCreator.createIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.DELETE, null ); assertEquals( task, queue.get( 0 ) ); task = - TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.FINISH ); + TaskCreator.createIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.FINISH, null ); assertEquals( task, queue.get( 1 ) ); } @@ -136,10 +136,10 @@ public class LuceneCleanupRemoveIndexedConsumerTest consumer.processArchivaArtifact( artifact ); repoFactoryControl.verify(); - + assertTrue( indexingQueue.getQueueSnapshot().isEmpty() ); } - + @Override protected String getPlexusConfigLocation() { diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumerTest.java b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumerTest.java index 918442493..1a5955327 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumerTest.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumerTest.java @@ -20,19 +20,23 @@ package org.apache.archiva.consumers.lucene; */ import java.io.File; +import java.io.IOException; import java.util.Calendar; import java.util.Date; -import java.util.List; +import java.util.HashSet; +import java.util.Set; import org.apache.commons.io.FileUtils; +import org.apache.maven.archiva.common.ArchivaException; import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer; import org.apache.maven.archiva.scheduled.ArchivaTaskScheduler; import org.apache.maven.archiva.scheduled.tasks.ArtifactIndexingTask; -import org.apache.maven.archiva.scheduled.tasks.TaskCreator; +import org.apache.maven.archiva.scheduled.tasks.DatabaseTask; +import org.apache.maven.archiva.scheduled.tasks.RepositoryTask; import org.codehaus.plexus.spring.PlexusInSpringTestCase; -import org.codehaus.plexus.taskqueue.TaskQueue; import org.codehaus.plexus.taskqueue.TaskQueueException; +import org.codehaus.plexus.taskqueue.execution.TaskExecutionException; /** * NexusIndexerConsumerTest @@ -40,11 +44,71 @@ import org.codehaus.plexus.taskqueue.TaskQueueException; public class NexusIndexerConsumerTest extends PlexusInSpringTestCase { + private final class ArchivaTaskSchedulerStub + implements ArchivaTaskScheduler + { + Set<File> indexed = new HashSet<File>(); + + public void startup() + throws ArchivaException + { + } + + public void scheduleDatabaseTasks() + throws TaskExecutionException + { + } + + public void queueRepositoryTask( RepositoryTask task ) + throws TaskQueueException + { + } + + public void queueIndexingTask( ArtifactIndexingTask task ) + throws TaskQueueException + { + switch ( task.getAction() ) + { + case ADD: + indexed.add( task.getResourceFile() ); + break; + case DELETE: + indexed.remove( task.getResourceFile() ); + break; + case FINISH: + try + { + task.getContext().close( false ); + } + catch ( IOException e ) + { + throw new TaskQueueException( e.getMessage() ); + } + break; + } + } + + public void queueDatabaseTask( DatabaseTask task ) + throws TaskQueueException + { + } + + public boolean isProcessingRepositoryTask( String repositoryId ) + { + return false; + } + + public boolean isProcessingDatabaseTask() + { + return false; + } + } + private KnownRepositoryContentConsumer nexusIndexerConsumer; private ManagedRepositoryConfiguration repositoryConfig; - private TaskQueue indexingQueue; + private ArchivaTaskSchedulerStub scheduler; @Override protected void setUp() @@ -52,9 +116,7 @@ public class NexusIndexerConsumerTest { super.setUp(); - ArchivaTaskScheduler scheduler = (ArchivaTaskScheduler) lookup( ArchivaTaskScheduler.class ); - - indexingQueue = (TaskQueue) lookup( TaskQueue.ROLE, "indexing" ); + scheduler = new ArchivaTaskSchedulerStub(); nexusIndexerConsumer = new NexusIndexerConsumer( scheduler ); @@ -97,21 +159,7 @@ public class NexusIndexerConsumerTest nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); nexusIndexerConsumer.completeScan(); - assertTaskQueued( artifactFile ); - } - - @SuppressWarnings( "unchecked" ) - private void assertTaskQueued( File artifactFile ) - throws TaskQueueException - { - List<ArtifactIndexingTask> queue = indexingQueue.getQueueSnapshot(); - assertEquals( 2, queue.size() ); - ArtifactIndexingTask task = - TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.ADD ); - assertEquals( task, queue.get( 0 ) ); - - task = TaskCreator.createIndexingTask( repositoryConfig.getId(), null, ArtifactIndexingTask.Action.FINISH ); - assertEquals( task, queue.get( 1 ) ); + assertTrue( scheduler.indexed.contains( artifactFile ) ); } public void testIndexerArtifactAlreadyIndexed() @@ -127,9 +175,7 @@ public class NexusIndexerConsumerTest nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); nexusIndexerConsumer.completeScan(); - assertTaskQueued( artifactFile ); - indexingQueue.take(); - indexingQueue.take(); + assertTrue( scheduler.indexed.contains( artifactFile ) ); // scan and index again now = Calendar.getInstance().getTime(); @@ -137,7 +183,7 @@ public class NexusIndexerConsumerTest nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); nexusIndexerConsumer.completeScan(); - assertTaskQueued( artifactFile ); + assertTrue( scheduler.indexed.contains( artifactFile ) ); } public void testIndexerIndexArtifactThenPom() @@ -153,9 +199,7 @@ public class NexusIndexerConsumerTest nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); nexusIndexerConsumer.completeScan(); - assertTaskQueued( artifactFile ); - indexingQueue.take(); - indexingQueue.take(); + assertTrue( scheduler.indexed.contains( artifactFile ) ); artifactFile = new File( repositoryConfig.getLocation(), "org/apache/archiva/archiva-index-methods-jar-test/1.0/pom.xml" ); @@ -166,7 +210,7 @@ public class NexusIndexerConsumerTest nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/pom.xml" ); nexusIndexerConsumer.completeScan(); - assertTaskQueued( artifactFile ); + assertTrue( scheduler.indexed.contains( artifactFile ) ); } @Override diff --git a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaIndexingTaskExecutor.java b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaIndexingTaskExecutor.java index a0f620c49..0bf8675b6 100644 --- a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaIndexingTaskExecutor.java +++ b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaIndexingTaskExecutor.java @@ -24,7 +24,6 @@ import java.io.IOException; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; -import org.apache.maven.archiva.configuration.ArchivaConfiguration; import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; import org.apache.maven.archiva.scheduled.tasks.ArtifactIndexingTask; import org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable; @@ -39,10 +38,7 @@ import org.sonatype.nexus.index.ArtifactContextProducer; import org.sonatype.nexus.index.ArtifactInfo; import org.sonatype.nexus.index.DefaultArtifactContextProducer; import org.sonatype.nexus.index.IndexerEngine; -import org.sonatype.nexus.index.NexusIndexer; -import org.sonatype.nexus.index.context.DefaultIndexingContext; import org.sonatype.nexus.index.context.IndexingContext; -import org.sonatype.nexus.index.context.UnsupportedExistingLuceneIndexException; import org.sonatype.nexus.index.packer.IndexPacker; import org.sonatype.nexus.index.packer.IndexPackingRequest; @@ -51,6 +47,7 @@ import org.sonatype.nexus.index.packer.IndexPackingRequest; * all performed by this executor. Add and update artifact in index tasks are added in the indexing task queue by the * NexusIndexerConsumer while remove artifact from index tasks are added by the LuceneCleanupRemoveIndexedConsumer. * + * @todo Nexus specifics shouldn't be in the archiva-scheduled module * @plexus.component role="org.codehaus.plexus.taskqueue.execution.TaskExecutor" role-hint="indexing" * instantiation-strategy="singleton" */ @@ -67,11 +64,6 @@ public class ArchivaIndexingTaskExecutor /** * @plexus.requirement */ - private ArchivaConfiguration archivaConfiguration; - - /** - * @plexus.requirement - */ private IndexPacker indexPacker; private ArtifactContextProducer artifactContextProducer; @@ -83,39 +75,47 @@ public class ArchivaIndexingTaskExecutor { ArtifactIndexingTask indexingTask = (ArtifactIndexingTask) task; - ManagedRepositoryConfiguration repository = - archivaConfiguration.getConfiguration().findManagedRepositoryById( indexingTask.getRepositoryId() ); - - String indexDir = repository.getIndexDir(); - File managedRepository = new File( repository.getLocation() ); - - File indexDirectory = null; - if ( indexDir != null && !"".equals( indexDir ) ) - { - indexDirectory = new File( repository.getIndexDir() ); - } - else - { - indexDirectory = new File( managedRepository, ".indexer" ); - } + ManagedRepositoryConfiguration repository = indexingTask.getRepository(); + IndexingContext context = indexingTask.getContext(); - IndexingContext context = null; - try + if ( ArtifactIndexingTask.Action.FINISH.equals( indexingTask.getAction() ) ) { - context = - new DefaultIndexingContext( repository.getId(), repository.getId(), managedRepository, - indexDirectory, null, null, NexusIndexer.FULL_INDEX, false ); - context.setSearchable( repository.isScanned() ); - - if ( ArtifactIndexingTask.Action.FINISH.equals( indexingTask.getAction() ) ) + try { + context.optimize(); + + File managedRepository = new File( repository.getLocation() ); final File indexLocation = new File( managedRepository, ".index" ); IndexPackingRequest request = new IndexPackingRequest( context, indexLocation ); indexPacker.packIndex( request ); - + log.debug( "Index file packaged at '" + indexLocation.getPath() + "'." ); } - else + catch ( IOException e ) + { + log.error( "Error occurred while executing indexing task '" + indexingTask + "'" ); + throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + + "'" ); + } + finally + { + if ( context != null ) + { + try + { + context.close( false ); + } + catch ( IOException e ) + { + log.error( "Error occurred while closing context: " + e.getMessage() ); + throw new TaskExecutionException( "Error occurred while closing context: " + e.getMessage() ); + } + } + } + } + else + { + try { File artifactFile = indexingTask.getResourceFile(); ArtifactContext ac = artifactContextProducer.getArtifactContext( context, artifactFile ); @@ -144,47 +144,28 @@ public class ArchivaIndexingTaskExecutor { log.debug( "Adding artifact '" + ac.getArtifactInfo() + "' to index.." ); indexerEngine.index( context, ac ); - context.optimize(); + context.getIndexWriter().commit(); } else { log.debug( "Updating artifact '" + ac.getArtifactInfo() + "' in index.." ); indexerEngine.update( context, ac ); - context.optimize(); + context.getIndexWriter().commit(); } } else { log.debug( "Removing artifact '" + ac.getArtifactInfo() + "' from index.." ); indexerEngine.remove( context, ac ); - context.optimize(); + context.getIndexWriter().commit(); } } } - } - catch ( IOException e ) - { - log.error( "Error occurred while executing indexing task '" + indexingTask + "'" ); - throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'" ); - } - catch ( UnsupportedExistingLuceneIndexException e ) - { - log.error( "Unsupported Lucene index format: " + e.getMessage() ); - throw new TaskExecutionException( "Unsupported Lucene index format: " + e.getMessage() ); - } - finally - { - if ( context != null ) + catch ( IOException e ) { - try - { - context.close( false ); - } - catch ( IOException e ) - { - log.error( "Error occurred while closing context: " + e.getMessage() ); - throw new TaskExecutionException( "Error occurred while closing context: " + e.getMessage() ); - } + log.error( "Error occurred while executing indexing task '" + indexingTask + "'" ); + throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + + "'" ); } } } @@ -207,10 +188,4 @@ public class ArchivaIndexingTaskExecutor { this.indexPacker = indexPacker; } - - public void setArchivaConfiguration( ArchivaConfiguration archivaConfiguration ) - { - this.archivaConfiguration = archivaConfiguration; - } - } diff --git a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/ArtifactIndexingTask.java b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/ArtifactIndexingTask.java index 91823c8b4..a9c0ac465 100644 --- a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/ArtifactIndexingTask.java +++ b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/ArtifactIndexingTask.java @@ -21,27 +21,33 @@ package org.apache.maven.archiva.scheduled.tasks; import java.io.File; +import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; import org.codehaus.plexus.taskqueue.Task; +import org.sonatype.nexus.index.context.IndexingContext; public class ArtifactIndexingTask implements Task { - public enum Action { ADD, DELETE, FINISH } - - private String repositoryId; - - private File resourceFile; - - private Action action; - - public String getRepositoryId() + public enum Action { - return repositoryId; + ADD, DELETE, FINISH } - public void setRepositoryId( String repositoryId ) + private final ManagedRepositoryConfiguration repository; + + private final File resourceFile; + + private final Action action; + + private final IndexingContext context; + + public ArtifactIndexingTask( ManagedRepositoryConfiguration repository, File resourceFile, Action action, + IndexingContext context ) { - this.repositoryId = repositoryId; + this.repository = repository; + this.resourceFile = resourceFile; + this.action = action; + this.context = context; } public long getMaxExecutionTime() @@ -54,26 +60,26 @@ public class ArtifactIndexingTask return resourceFile; } - public void setResourceFile( File resourceFile ) + public Action getAction() { - this.resourceFile = resourceFile; + return action; } - public Action getAction() + @Override + public String toString() { - return action; + return "ArtifactIndexingTask [action=" + action + ", repositoryId=" + repository.getId() + ", resourceFile=" + + resourceFile + "]"; } - public void setAction( Action action ) + public ManagedRepositoryConfiguration getRepository() { - this.action = action; + return repository; } - @Override - public String toString() + public IndexingContext getContext() { - return "ArtifactIndexingTask [action=" + action + ", repositoryId=" + repositoryId + ", resourceFile=" - + resourceFile + "]"; + return context; } @Override @@ -81,8 +87,8 @@ public class ArtifactIndexingTask { final int prime = 31; int result = 1; - result = prime * result + ( ( action == null ) ? 0 : action.hashCode() ); - result = prime * result + ( ( repositoryId == null ) ? 0 : repositoryId.hashCode() ); + result = prime * result + action.hashCode(); + result = prime * result + repository.getId().hashCode(); result = prime * result + ( ( resourceFile == null ) ? 0 : resourceFile.hashCode() ); return result; } @@ -97,19 +103,9 @@ public class ArtifactIndexingTask if ( getClass() != obj.getClass() ) return false; ArtifactIndexingTask other = (ArtifactIndexingTask) obj; - if ( action == null ) - { - if ( other.action != null ) - return false; - } - else if ( !action.equals( other.action ) ) + if ( !action.equals( other.action ) ) return false; - if ( repositoryId == null ) - { - if ( other.repositoryId != null ) - return false; - } - else if ( !repositoryId.equals( other.repositoryId ) ) + if ( !repository.getId().equals( other.repository.getId() ) ) return false; if ( resourceFile == null ) { diff --git a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/TaskCreator.java b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/TaskCreator.java index 8230deea4..5075b6e81 100644 --- a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/TaskCreator.java +++ b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/TaskCreator.java @@ -20,9 +20,17 @@ package org.apache.maven.archiva.scheduled.tasks; */ import java.io.File; +import java.io.IOException; + +import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; +import org.sonatype.nexus.index.NexusIndexer; +import org.sonatype.nexus.index.context.DefaultIndexingContext; +import org.sonatype.nexus.index.context.IndexingContext; +import org.sonatype.nexus.index.context.UnsupportedExistingLuceneIndexException; /** * TaskCreator Convenience class for creating Archiva tasks. + * @todo Nexus specifics shouldn't be in the archiva-scheduled module */ public class TaskCreator { @@ -60,15 +68,33 @@ public class TaskCreator return task; } - public static ArtifactIndexingTask createIndexingTask( String repositoryId, File resource, - ArtifactIndexingTask.Action action ) + public static ArtifactIndexingTask createIndexingTask( ManagedRepositoryConfiguration repository, File resource, + ArtifactIndexingTask.Action action, IndexingContext context ) { - ArtifactIndexingTask task = new ArtifactIndexingTask(); - task.setRepositoryId( repositoryId ); - task.setAction( action ); - task.setResourceFile( resource ); + return new ArtifactIndexingTask( repository, resource, action, context ); + } - return task; + public static IndexingContext createContext( ManagedRepositoryConfiguration repository ) + throws IOException, UnsupportedExistingLuceneIndexException + { + String indexDir = repository.getIndexDir(); + File managedRepository = new File( repository.getLocation() ); + + File indexDirectory = null; + if ( indexDir != null && !"".equals( indexDir ) ) + { + indexDirectory = new File( repository.getIndexDir() ); + } + else + { + indexDirectory = new File( managedRepository, ".indexer" ); + } + + IndexingContext context = + new DefaultIndexingContext( repository.getId(), repository.getId(), managedRepository, indexDirectory, + null, null, NexusIndexer.FULL_INDEX, false ); + context.setSearchable( repository.isScanned() ); + return context; } } diff --git a/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaIndexingTaskExecutorTest.java b/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaIndexingTaskExecutorTest.java index 0d0de4d11..af20d0868 100644 --- a/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaIndexingTaskExecutorTest.java +++ b/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaIndexingTaskExecutorTest.java @@ -35,13 +35,11 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.maven.archiva.configuration.ArchivaConfiguration; import org.apache.maven.archiva.configuration.Configuration; import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; import org.apache.maven.archiva.scheduled.tasks.ArtifactIndexingTask; import org.apache.maven.archiva.scheduled.tasks.TaskCreator; import org.codehaus.plexus.spring.PlexusInSpringTestCase; -import org.easymock.MockControl; import org.sonatype.nexus.index.ArtifactInfo; import org.sonatype.nexus.index.FlatSearchRequest; import org.sonatype.nexus.index.FlatSearchResponse; @@ -62,16 +60,14 @@ public class ArchivaIndexingTaskExecutorTest private IndexPacker indexPacker; - private MockControl archivaConfigControl; - - private ArchivaConfiguration archivaConfiguration; - private ManagedRepositoryConfiguration repositoryConfig; private Configuration configuration; private NexusIndexer indexer; + private IndexingContext context; + protected void setUp() throws Exception { @@ -92,21 +88,21 @@ public class ArchivaIndexingTaskExecutorTest configuration = new Configuration(); configuration.addManagedRepository( repositoryConfig ); - archivaConfigControl = MockControl.createControl( ArchivaConfiguration.class ); - archivaConfiguration = (ArchivaConfiguration) archivaConfigControl.getMock(); - indexer = (NexusIndexer) lookup( NexusIndexer.class ); indexerEngine = (IndexerEngine) lookup( IndexerEngine.class ); indexPacker = (IndexPacker) lookup( IndexPacker.class ); indexingExecutor.setIndexerEngine( indexerEngine ); indexingExecutor.setIndexPacker( indexPacker ); - indexingExecutor.setArchivaConfiguration( archivaConfiguration ); + + context = TaskCreator.createContext( repositoryConfig ); } protected void tearDown() throws Exception { + context.close( false ); + // delete created index in the repository File indexDir = new File( repositoryConfig.getLocation(), ".indexer" ); FileUtils.deleteDirectory( indexDir ); @@ -127,16 +123,10 @@ public class ArchivaIndexingTaskExecutorTest "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); ArtifactIndexingTask task = - TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.ADD ); - - archivaConfigControl.expectAndReturn( archivaConfiguration.getConfiguration(), configuration ); - - archivaConfigControl.replay(); + TaskCreator.createIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.ADD, context ); indexingExecutor.executeTask( task ); - archivaConfigControl.verify(); - BooleanQuery q = new BooleanQuery(); q.add( indexer.constructQuery( ArtifactInfo.GROUP_ID, "org.apache.archiva" ), Occur.SHOULD ); q.add( indexer.constructQuery( ArtifactInfo.ARTIFACT_ID, "archiva-index-methods-jar-test" ), Occur.SHOULD ); @@ -171,17 +161,11 @@ public class ArchivaIndexingTaskExecutorTest "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); ArtifactIndexingTask task = - TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.ADD ); - - archivaConfigControl.expectAndReturn( archivaConfiguration.getConfiguration(), configuration, 2 ); - - archivaConfigControl.replay(); + TaskCreator.createIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.ADD, context ); indexingExecutor.executeTask( task ); indexingExecutor.executeTask( task ); - archivaConfigControl.verify(); - BooleanQuery q = new BooleanQuery(); q.add( indexer.constructQuery( ArtifactInfo.GROUP_ID, "org.apache.archiva" ), Occur.SHOULD ); q.add( indexer.constructQuery( ArtifactInfo.ARTIFACT_ID, "archiva-index-methods-jar-test" ), Occur.SHOULD ); @@ -204,18 +188,9 @@ public class ArchivaIndexingTaskExecutorTest "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); ArtifactIndexingTask task = - TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.ADD ); - - archivaConfigControl.expectAndReturn( archivaConfiguration.getConfiguration(), configuration, 3 ); - - archivaConfigControl.replay(); + TaskCreator.createIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.ADD, context ); - // add artifact to index - indexingExecutor.executeTask( task ); - - task = TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.FINISH ); - - // add artifact to index + // remove artifact from index indexingExecutor.executeTask( task ); BooleanQuery q = new BooleanQuery(); @@ -226,16 +201,21 @@ public class ArchivaIndexingTaskExecutorTest TopDocs topDocs = searcher.search( q, null, 10 ); assertTrue( new File( repositoryConfig.getLocation(), ".indexer" ).exists() ); - assertTrue( new File( repositoryConfig.getLocation(), ".index" ).exists() ); + assertFalse( new File( repositoryConfig.getLocation(), ".index" ).exists() ); // should return 1 hit assertEquals( 1, topDocs.totalHits ); + context = TaskCreator.createContext( repositoryConfig ); + // remove added artifact from index - task = TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.DELETE ); + task = + TaskCreator.createIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.DELETE, context ); indexingExecutor.executeTask( task ); - archivaConfigControl.verify(); + task = + TaskCreator.createIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.FINISH, context ); + indexingExecutor.executeTask( task ); q = new BooleanQuery(); q.add( indexer.constructQuery( ArtifactInfo.GROUP_ID, "org.apache.archiva" ), Occur.SHOULD ); @@ -249,7 +229,7 @@ public class ArchivaIndexingTaskExecutorTest // artifact should have been removed from the index! assertEquals( 0, topDocs.totalHits ); - + // TODO: test it was removed from the packaged index also } @@ -261,28 +241,15 @@ public class ArchivaIndexingTaskExecutorTest "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ); ArtifactIndexingTask task = - TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.ADD ); - - archivaConfigControl.expectAndReturn( archivaConfiguration.getConfiguration(), configuration ); - - archivaConfigControl.replay(); + TaskCreator.createIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.ADD, context ); indexingExecutor.executeTask( task ); - archivaConfigControl.verify(); - - archivaConfigControl.reset(); - - task = TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.FINISH ); - - archivaConfigControl.expectAndReturn( archivaConfiguration.getConfiguration(), configuration ); - - archivaConfigControl.replay(); + task = + TaskCreator.createIndexingTask( repositoryConfig, artifactFile, ArtifactIndexingTask.Action.FINISH, context ); indexingExecutor.executeTask( task ); - archivaConfigControl.verify(); - assertTrue( new File( repositoryConfig.getLocation(), ".indexer" ).exists() ); assertTrue( new File( repositoryConfig.getLocation(), ".index" ).exists() ); diff --git a/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/tasks/TaskCreatorTest.java b/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/tasks/TaskCreatorTest.java index baab0ab65..68c1ad476 100644 --- a/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/tasks/TaskCreatorTest.java +++ b/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/tasks/TaskCreatorTest.java @@ -84,17 +84,6 @@ public class TaskCreatorTest assertFalse( task.isUpdateRelatedArtifacts() ); } - public void testCreateIndexingTask() - throws Exception - { - File resource = new File( getBasedir(), "target/test-classes/test.jar" ); - ArtifactIndexingTask task = TaskCreator.createIndexingTask( REPO_ID, resource, ArtifactIndexingTask.Action.ADD ); - - assertEquals( "Incorrect repository id set.", REPO_ID, task.getRepositoryId() ); - assertEquals( "Incorrect action set.", ArtifactIndexingTask.Action.ADD, task.getAction() ); - assertEquals( "Incorrect resource file set.", resource, task.getResourceFile() ); - } - private void assertBasicTaskDetails( RepositoryTask task ) { assertEquals( "Incorrect repository id set.", REPO_ID, task.getRepositoryId() ); |