[MRM-1264] reduce memory used by indexing process

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@825082 13f79535-47bb-0310-9956-ffa450edef68

4 files changed, 90 insertions, 71 deletions

diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumer.java
index 4ad4c2f7d..ef310b04f 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumer.java
@@ -19,6 +19,9 @@ package org.apache.archiva.consumers.lucene;
  * under the License.
  */
 
+import java.io.File;
+import java.util.List;
+
 import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
 import org.apache.maven.archiva.consumers.AbstractMonitoredConsumer;
 import org.apache.maven.archiva.consumers.ConsumerException;
@@ -34,9 +37,6 @@ import org.codehaus.plexus.taskqueue.TaskQueueException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.util.List;
-
 /**
  * LuceneCleanupRemoveIndexedConsumer
  * 
@@ -97,16 +97,16 @@ public class LuceneCleanupRemoveIndexedConsumer
             if ( !artifactFile.exists() )
             {
                 ArtifactIndexingTask task =                
-                        TaskCreator.createIndexingTask( repository.getId(), artifactFile, ArtifactIndexingTask.DELETE );
+                        TaskCreator.createIndexingTask( repository.getId(), artifactFile, ArtifactIndexingTask.Action.DELETE );
                 
-                log.debug( "Queueing indexing task '" + task.getName() + "' to remove the artifact from the index." );
+                log.debug( "Queueing indexing task '" + task + "' to remove the artifact from the index." );
                 scheduler.queueIndexingTask( task );
 
                 // note we finish immediately here since it isn't done repo-by-repo. It might be nice to ensure that is
                 // the case for optimisation though
                 task =
-                    TaskCreator.createIndexingTask( repository.getId(), artifactFile, ArtifactIndexingTask.FINISH );
-                log.debug( "Queueing indexing task + '" + task.getName() + "' to finish indexing." );
+                    TaskCreator.createIndexingTask( repository.getId(), artifactFile, ArtifactIndexingTask.Action.FINISH );
+                log.debug( "Queueing indexing task + '" + task + "' to finish indexing." );
                 scheduler.queueIndexingTask( task );
             }
                    
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java
index 69ce89f48..94a22996f 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java
@@ -33,6 +33,7 @@ import org.apache.maven.archiva.repository.content.ManagedDefaultRepositoryConte
 import org.apache.maven.archiva.scheduled.ArchivaTaskScheduler;
 import org.apache.maven.archiva.scheduled.tasks.ArtifactIndexingTask;
 import org.apache.maven.archiva.scheduled.tasks.TaskCreator;
+import org.apache.maven.archiva.scheduled.tasks.ArtifactIndexingTask.Action;
 import org.codehaus.plexus.taskqueue.TaskQueueException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -90,10 +91,10 @@ public class NexusIndexerConsumer
         File artifactFile = new File( managedRepository, path );
                 
         ArtifactIndexingTask task =
-            TaskCreator.createIndexingTask( repositoryContent.getId(), artifactFile, ArtifactIndexingTask.ADD );
+            TaskCreator.createIndexingTask( repositoryContent.getId(), artifactFile, ArtifactIndexingTask.Action.ADD );
         try
         {
-            log.debug( "Queueing indexing task + '" + task.getName() + "' to add or update the artifact in the index." );
+            log.debug( "Queueing indexing task + '" + task + "' to add or update the artifact in the index." );
             scheduler.queueIndexingTask( task );
         }
         catch ( TaskQueueException e )
@@ -105,15 +106,15 @@ public class NexusIndexerConsumer
     public void completeScan()
     {
         ArtifactIndexingTask task =
-            TaskCreator.createIndexingTask( repositoryContent.getId(), null, ArtifactIndexingTask.FINISH );
+            TaskCreator.createIndexingTask( repositoryContent.getId(), null, ArtifactIndexingTask.Action.FINISH );
         try
         {
-            log.debug( "Queueing indexing task + '" + task.getName() + "' to finish indexing." );
+            log.debug( "Queueing indexing task + '" + task + "' to finish indexing." );
             scheduler.queueIndexingTask( task );
         }
         catch ( TaskQueueException e )
         {
-            log.error( "Error queueing task: " + task.getName() + ": " + e.getMessage(), e );
+            log.error( "Error queueing task: " + task + ": " + e.getMessage(), e );
         }        
     }
 
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumerTest.java b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumerTest.java
index 188735981..4f005dff2 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumerTest.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/LuceneCleanupRemoveIndexedConsumerTest.java
@@ -20,6 +20,7 @@ package org.apache.archiva.consumers.lucene;
  */
 
 import java.io.File;
+import java.util.List;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
@@ -31,6 +32,7 @@ import org.apache.maven.archiva.scheduled.ArchivaTaskScheduler;
 import org.apache.maven.archiva.scheduled.tasks.ArtifactIndexingTask;
 import org.apache.maven.archiva.scheduled.tasks.TaskCreator;
 import org.codehaus.plexus.spring.PlexusInSpringTestCase;
+import org.codehaus.plexus.taskqueue.TaskQueue;
 import org.easymock.MockControl;
 import org.easymock.classextension.MockClassControl;
 
@@ -48,7 +50,7 @@ public class LuceneCleanupRemoveIndexedConsumerTest
 
     private ManagedRepositoryConfiguration repositoryConfig;
     
-    private ArchivaTaskScheduler scheduler;
+    private TaskQueue indexingQueue;
 
     public void setUp()
         throws Exception
@@ -58,7 +60,9 @@ public class LuceneCleanupRemoveIndexedConsumerTest
         repoFactoryControl = MockClassControl.createControl( RepositoryContentFactory.class );
         repoFactory = (RepositoryContentFactory) repoFactoryControl.getMock();
 
-        scheduler = ( ArchivaTaskScheduler ) lookup( ArchivaTaskScheduler.class );
+        ArchivaTaskScheduler scheduler = ( ArchivaTaskScheduler ) lookup( ArchivaTaskScheduler.class );
+        
+        indexingQueue = (TaskQueue) lookup( TaskQueue.ROLE, "indexing" );
         
         consumer = new LuceneCleanupRemoveIndexedConsumer( repoFactory, scheduler );
 
@@ -81,9 +85,12 @@ public class LuceneCleanupRemoveIndexedConsumerTest
         super.tearDown();
     }
 
+    @SuppressWarnings( "unchecked" )
     public void testProcessArtifactArtifactDoesNotExist()
         throws Exception
     {
+        assertTrue( indexingQueue.getQueueSnapshot().isEmpty() );
+
         ArchivaArtifact artifact =
             new ArchivaArtifact( "org.apache.archiva", "archiva-lucene-consumers", "1.2", null, "jar", "test-repo" );
         
@@ -92,9 +99,6 @@ public class LuceneCleanupRemoveIndexedConsumerTest
         
         File artifactFile = new File( repoContent.getRepoRoot(), repoContent.toPath( artifact ) );
        
-        ArtifactIndexingTask task =
-            TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.DELETE );
-        
         repoFactoryControl.expectAndReturn( repoFactory.getManagedRepositoryContent( repositoryConfig.getId() ),
                                             repoContent );
        
@@ -104,22 +108,26 @@ public class LuceneCleanupRemoveIndexedConsumerTest
 
         repoFactoryControl.verify();       
         
-        assertTrue( scheduler.isProcessingIndexingTaskWithName( task.getName() ) );               
+        List<ArtifactIndexingTask> queue = indexingQueue.getQueueSnapshot();
+        assertEquals( 2, queue.size() );
+        ArtifactIndexingTask task =
+            TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.DELETE );
+        assertEquals( task, queue.get( 0 ) );
+        task =
+            TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.FINISH );
+        assertEquals( task, queue.get( 1 ) );
     }
 
     public void testProcessArtifactArtifactExists()
         throws Exception
     {
+        assertTrue( indexingQueue.getQueueSnapshot().isEmpty() );
+
         ArchivaArtifact artifact =
             new ArchivaArtifact( "org.apache.maven.archiva", "archiva-lucene-cleanup", "1.0", null, "jar", "test-repo" );
         ManagedRepositoryContent repoContent = new ManagedDefaultRepositoryContent();
         repoContent.setRepository( repositoryConfig );
 
-        File artifactFile = new File( repoContent.getRepoRoot(), repoContent.toPath( artifact ) );
-        
-        ArtifactIndexingTask task =
-            TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.DELETE );
-        
         repoFactoryControl.expectAndReturn( repoFactory.getManagedRepositoryContent( repositoryConfig.getId() ),
                                             repoContent );
 
@@ -129,7 +137,7 @@ public class LuceneCleanupRemoveIndexedConsumerTest
 
         repoFactoryControl.verify();
         
-        assertFalse( scheduler.isProcessingIndexingTaskWithName( task.getName() ) );
+        assertTrue( indexingQueue.getQueueSnapshot().isEmpty() );
     }
     
     @Override
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumerTest.java b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumerTest.java
index 2bb33524d..918442493 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumerTest.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumerTest.java
@@ -22,6 +22,7 @@ package org.apache.archiva.consumers.lucene;
 import java.io.File;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.List;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
@@ -30,6 +31,8 @@ import org.apache.maven.archiva.scheduled.ArchivaTaskScheduler;
 import org.apache.maven.archiva.scheduled.tasks.ArtifactIndexingTask;
 import org.apache.maven.archiva.scheduled.tasks.TaskCreator;
 import org.codehaus.plexus.spring.PlexusInSpringTestCase;
+import org.codehaus.plexus.taskqueue.TaskQueue;
+import org.codehaus.plexus.taskqueue.TaskQueueException;
 
 /**
  * NexusIndexerConsumerTest
@@ -38,21 +41,23 @@ public class NexusIndexerConsumerTest
     extends PlexusInSpringTestCase
 {
     private KnownRepositoryContentConsumer nexusIndexerConsumer;
-        
+
     private ManagedRepositoryConfiguration repositoryConfig;
-    
-    private ArchivaTaskScheduler scheduler;
-    
+
+    private TaskQueue indexingQueue;
+
     @Override
-    protected void setUp() 
+    protected void setUp()
         throws Exception
     {
         super.setUp();
-        
-        scheduler = ( ArchivaTaskScheduler ) lookup( ArchivaTaskScheduler.class );
-        
+
+        ArchivaTaskScheduler scheduler = (ArchivaTaskScheduler) lookup( ArchivaTaskScheduler.class );
+
+        indexingQueue = (TaskQueue) lookup( TaskQueue.ROLE, "indexing" );
+
         nexusIndexerConsumer = new NexusIndexerConsumer( scheduler );
-                
+
         repositoryConfig = new ManagedRepositoryConfiguration();
         repositoryConfig.setId( "test-repo" );
         repositoryConfig.setLocation( getBasedir() + "/target/test-classes/test-repo" );
@@ -62,7 +67,7 @@ public class NexusIndexerConsumerTest
         repositoryConfig.setSnapshots( false );
         repositoryConfig.setReleases( true );
     }
-    
+
     @Override
     protected void tearDown()
         throws Exception
@@ -71,33 +76,44 @@ public class NexusIndexerConsumerTest
         File indexDir = new File( repositoryConfig.getLocation(), ".indexer" );
         FileUtils.deleteDirectory( indexDir );
         assertFalse( indexDir.exists() );
-        
+
         indexDir = new File( repositoryConfig.getLocation(), ".index" );
         FileUtils.deleteDirectory( indexDir );
         assertFalse( indexDir.exists() );
-        
+
         super.tearDown();
     }
-    
+
     public void testIndexerIndexArtifact()
         throws Exception
-    {        
+    {
         File artifactFile =
             new File( repositoryConfig.getLocation(),
                       "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" );
 
-        ArtifactIndexingTask task =
-            TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.ADD );
-        
         // begin scan
         Date now = Calendar.getInstance().getTime();
         nexusIndexerConsumer.beginScan( repositoryConfig, now );
         nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" );
         nexusIndexerConsumer.completeScan();
-                
-        assertTrue( scheduler.isProcessingIndexingTaskWithName( task.getName() ) );        
+
+        assertTaskQueued( artifactFile );
+    }
+
+    @SuppressWarnings( "unchecked" )
+    private void assertTaskQueued( File artifactFile )
+        throws TaskQueueException
+    {
+        List<ArtifactIndexingTask> queue = indexingQueue.getQueueSnapshot();
+        assertEquals( 2, queue.size() );
+        ArtifactIndexingTask task =
+            TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.Action.ADD );
+        assertEquals( task, queue.get( 0 ) );
+
+        task = TaskCreator.createIndexingTask( repositoryConfig.getId(), null, ArtifactIndexingTask.Action.FINISH );
+        assertEquals( task, queue.get( 1 ) );
     }
-    
+
     public void testIndexerArtifactAlreadyIndexed()
         throws Exception
     {
@@ -105,60 +121,54 @@ public class NexusIndexerConsumerTest
             new File( repositoryConfig.getLocation(),
                       "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" );
 
-        ArtifactIndexingTask task =
-            TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.ADD );
-        
         // begin scan
         Date now = Calendar.getInstance().getTime();
         nexusIndexerConsumer.beginScan( repositoryConfig, now );
         nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" );
         nexusIndexerConsumer.completeScan();
-        
-        assertTrue( scheduler.isProcessingIndexingTaskWithName( task.getName() ) );
-        
+
+        assertTaskQueued( artifactFile );
+        indexingQueue.take();
+        indexingQueue.take();
+
         // scan and index again
         now = Calendar.getInstance().getTime();
         nexusIndexerConsumer.beginScan( repositoryConfig, now );
-        nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" );        
+        nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" );
         nexusIndexerConsumer.completeScan();
-        
-        assertTrue( scheduler.isProcessingIndexingTaskWithName( task.getName() ) );        
+
+        assertTaskQueued( artifactFile );
     }
-    
+
     public void testIndexerIndexArtifactThenPom()
         throws Exception
-    {    
+    {
         File artifactFile =
             new File( repositoryConfig.getLocation(),
                       "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" );
 
-        ArtifactIndexingTask task =
-            TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.ADD );
-        
         // begin scan
         Date now = Calendar.getInstance().getTime();
         nexusIndexerConsumer.beginScan( repositoryConfig, now );
         nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" );
         nexusIndexerConsumer.completeScan();
-        
-        assertTrue( scheduler.isProcessingIndexingTaskWithName( task.getName() ) );
-        
+
+        assertTaskQueued( artifactFile );
+        indexingQueue.take();
+        indexingQueue.take();
+
         artifactFile =
-            new File( repositoryConfig.getLocation(),
-                      "org/apache/archiva/archiva-index-methods-jar-test/1.0/pom.xml" );
+            new File( repositoryConfig.getLocation(), "org/apache/archiva/archiva-index-methods-jar-test/1.0/pom.xml" );
 
-        task =
-            TaskCreator.createIndexingTask( repositoryConfig.getId(), artifactFile, ArtifactIndexingTask.ADD );
-        
         // scan and index again
         now = Calendar.getInstance().getTime();
         nexusIndexerConsumer.beginScan( repositoryConfig, now );
-        nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/pom.xml" );        
+        nexusIndexerConsumer.processFile( "org/apache/archiva/archiva-index-methods-jar-test/1.0/pom.xml" );
         nexusIndexerConsumer.completeScan();
-        
-        assertTrue( scheduler.isProcessingIndexingTaskWithName( task.getName() ) );
-    }  
-    
+
+        assertTaskQueued( artifactFile );
+    }
+
     @Override
     protected String getPlexusConfigLocation()
     {