[MRM-1097]

o upgrade nexus indexer to 3.0.1 + added logger bean which is needed by NexusIndexer
o for deployments/proxied artifacts, create and close indexing context when indexing task is executed as compared to the original behavior where the indexing context is created 
before it is queued (so as the indexing task queue builds up, more indexing contexts are being opened/created which might be causing the too many open files problem) 
o updated unit tests with changes made


git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1039004 13f79535-47bb-0310-9956-ffa450edef68

11 files changed, 234 insertions, 0 deletions

diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-consumer-api/src/main/java/org/apache/maven/archiva/consumers/RepositoryContentConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-consumer-api/src/main/java/org/apache/maven/archiva/consumers/RepositoryContentConsumer.java
index 203e5bd76..f7b5a2487 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-consumer-api/src/main/java/org/apache/maven/archiva/consumers/RepositoryContentConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-consumer-api/src/main/java/org/apache/maven/archiva/consumers/RepositoryContentConsumer.java
@@ -63,6 +63,22 @@ public interface RepositoryContentConsumer extends Consumer
 
     /**
      * <p>
+     * Event that triggers at the beginning of a scan, where you can also indicate whether the consumers will be
+     * executed on an entire repository or on a specific resource.
+     * </p>
+     *
+     * @see RepositoryContentConsumer#beginScan(org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration, java.util.Date )
+     *
+     * @param repository the repository that this consumer is being used for.
+     * @param whenGathered the start of the repository scan
+     * @param executeOnEntireRepo flags whether the consumer will be executed on an entire repository or just on a specific resource
+     * @throws ConsumerException if there was a problem with using the provided repository with the consumer.
+     */
+    public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException;
+
+    /**
+     * <p>
      * Event indicating a file is to be processed by this consumer.
      * </p> 
      * 
@@ -76,6 +92,14 @@ public interface RepositoryContentConsumer extends Consumer
      * @throws ConsumerException if there was a problem processing this file.
      */
     public void processFile( String path ) throws ConsumerException;
+
+    /**
+     *
+     * @param path
+     * @param executeOnEntireRepo
+     * @throws Exception
+     */
+    public void processFile( String path, boolean executeOnEntireRepo ) throws Exception;
     
     /**
      * <p>
@@ -90,6 +114,13 @@ public interface RepositoryContentConsumer extends Consumer
     public void completeScan();
 
     /**
+     * 
+     * @param executeOnEntireRepo
+     * @throws Exception
+     */
+    public void completeScan( boolean executeOnEntireRepo );
+
+    /**
      * Whether the consumer should process files that have not been modified since the time passed in to the scan
      * method.
      * @return whether to process the unmodified files
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java
index fa2e28987..e01510882 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java
@@ -99,11 +99,22 @@ public class ArtifactMissingChecksumsConsumer
         this.repositoryDir = new File( repo.getLocation() );
     }
 
+    public void beginScan( ManagedRepositoryConfiguration repo, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        beginScan( repo, whenGathered );
+    }
+
     public void completeScan()
     {
         /* do nothing */
     }
 
+    public void completeScan( boolean executeOnEntireRepo )
+    {
+        completeScan();
+    }
+
     public List<String> getExcludes()
     {
         return getDefaultArtifactExclusions();
@@ -120,6 +131,12 @@ public class ArtifactMissingChecksumsConsumer
         createFixChecksum( path, new ChecksumAlgorithm[] { ChecksumAlgorithm.SHA1 } );
         createFixChecksum( path, new ChecksumAlgorithm[] { ChecksumAlgorithm.MD5 } );        
     }
+
+    public void processFile( String path, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        processFile( path );   
+    }
     
     private void createFixChecksum( String path, ChecksumAlgorithm checksumAlgorithm[] )
     {
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/AutoRemoveConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/AutoRemoveConsumer.java
index 9d39e8c2b..f510b81df 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/AutoRemoveConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/AutoRemoveConsumer.java
@@ -95,11 +95,22 @@ public class AutoRemoveConsumer
         this.repositoryDir = new File( repository.getLocation() );
     }
 
+    public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        beginScan( repository, whenGathered );
+    }
+
     public void completeScan()
     {
         /* do nothing */
     }
 
+    public void completeScan( boolean executeOnEntireRepo )
+    {
+        completeScan();
+    }
+
     public List<String> getExcludes()
     {
         return null;
@@ -121,6 +132,12 @@ public class AutoRemoveConsumer
         }
     }
 
+    public void processFile( String path, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        processFile( path );    
+    }
+
     public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
     {                
         if ( ConfigurationNames.isRepositoryScanning( propertyName ) )
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/AutoRenameConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/AutoRenameConsumer.java
index 2df8820db..3fb70b961 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/AutoRenameConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/AutoRenameConsumer.java
@@ -96,11 +96,22 @@ public class AutoRenameConsumer
         this.repositoryDir = new File( repository.getLocation() );
     }
 
+    public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        beginScan( repository, whenGathered );
+    }
+
     public void completeScan()
     {
         /* do nothing */
     }
 
+    public void completeScan( boolean executeOnEntireRepo )
+    {
+        completeScan();
+    }
+
     public List<String> getExcludes()
     {
         return null;
@@ -143,4 +154,10 @@ public class AutoRenameConsumer
             file.delete();
         }
     }
+
+    public void processFile( String path, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        processFile( path );        
+    }
 }
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/MetadataUpdaterConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/MetadataUpdaterConsumer.java
index e40bb8d27..c8d0ddc0a 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/MetadataUpdaterConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/MetadataUpdaterConsumer.java
@@ -142,11 +142,24 @@ public class MetadataUpdaterConsumer
         }
     }
 
+    @Override
+    public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        beginScan( repository, whenGathered );
+    }
+
     public void completeScan()
     {
         /* do nothing here */
     }
 
+    @Override
+    public void completeScan( boolean executeOnEntireRepo )
+    {
+        completeScan();
+    }
+
     public List<String> getExcludes()
     {
         return getDefaultArtifactExclusions();
@@ -176,6 +189,13 @@ public class MetadataUpdaterConsumer
         }
     }
 
+    @Override
+    public void processFile( String path, boolean executeOnEntireRepo )
+        throws Exception
+    {
+        processFile( path );
+    }
+
     private void updateProjectMetadata( ArtifactReference artifact, String path )
     {
         ProjectReference projectRef = new ProjectReference();
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ValidateChecksumConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ValidateChecksumConsumer.java
index e7ac88799..051f8e514 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ValidateChecksumConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ValidateChecksumConsumer.java
@@ -102,11 +102,24 @@ public class ValidateChecksumConsumer
         this.repositoryDir = new File( repository.getLocation() );
     }
 
+    @Override
+    public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        beginScan( repository, whenGathered );
+    }
+
     public void completeScan()
     {
         /* nothing to do */
     }
 
+    @Override
+    public void completeScan( boolean executeOnEntireRepo )
+    {
+        completeScan();
+    }
+
     public List<String> getExcludes()
     {
         return null;
@@ -143,6 +156,13 @@ public class ValidateChecksumConsumer
         }
     }
 
+    @Override
+    public void processFile( String path, boolean executeOnEntireRepo )
+        throws Exception
+    {
+        processFile( path );
+    }
+
     public void initialize()
         throws InitializationException
     {
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/repository/RepositoryPurgeConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/repository/RepositoryPurgeConsumer.java
index bcdc5c5fa..a77507cf7 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/repository/RepositoryPurgeConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/repository/RepositoryPurgeConsumer.java
@@ -157,6 +157,13 @@ public class RepositoryPurgeConsumer
         }
     }
 
+    @Override
+    public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        beginScan( repository, whenGathered );
+    }
+
     public void processFile( String path )
         throws ConsumerException
     {
@@ -175,11 +182,24 @@ public class RepositoryPurgeConsumer
         }
     }
 
+    @Override
+    public void processFile( String path, boolean executeOnEntireRepo )
+        throws Exception
+    {
+        processFile( path );
+    }
+
     public void completeScan()
     {
         /* do nothing */
     }
 
+    @Override
+    public void completeScan( boolean executeOnEntireRepo )
+    {
+        completeScan();
+    }
+
     public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
     {
         if ( ConfigurationNames.isRepositoryScanning( propertyName ) )
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-dependency-tree-consumer/src/main/java/org/apache/archiva/consumers/dependencytree/DependencyTreeGeneratorConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-dependency-tree-consumer/src/main/java/org/apache/archiva/consumers/dependencytree/DependencyTreeGeneratorConsumer.java
index 75772566e..973e71f67 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-dependency-tree-consumer/src/main/java/org/apache/archiva/consumers/dependencytree/DependencyTreeGeneratorConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-dependency-tree-consumer/src/main/java/org/apache/archiva/consumers/dependencytree/DependencyTreeGeneratorConsumer.java
@@ -155,10 +155,21 @@ public class DependencyTreeGeneratorConsumer
         }
     }
 
+    public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        beginScan( repository, whenGathered );    
+    }
+
     public void completeScan()
     {
     }
 
+    public void completeScan( boolean executeOnEntireRepo )
+    {
+        completeScan();
+    }
+
     public List<String> getExcludes()
     {
         return null;
@@ -246,6 +257,12 @@ public class DependencyTreeGeneratorConsumer
         }
     }
 
+    public void processFile( String path, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        processFile( path );    
+    }
+
     private void flushProjectCache( MavenProjectBuilder projectBuilder )
     {
         try
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java
index 836fa6a8e..5e1f1ee9c 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/java/org/apache/archiva/consumers/lucene/NexusIndexerConsumer.java
@@ -99,6 +99,7 @@ public class NexusIndexerConsumer
 
         try
         {
+            log.info( "Creating indexing context for repo : " + repository.getId() );
             context = ArtifactIndexingTask.createContext( repository );
         }
         catch ( IOException e )
@@ -111,6 +112,21 @@ public class NexusIndexerConsumer
         }
     }
 
+    @Override
+    public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        if( executeOnEntireRepo )
+        {
+            beginScan( repository, whenGathered );
+        }
+        else
+        {
+            this.repository = repository;
+            managedRepository = new File( repository.getLocation() );       
+        }
+    }
+
     public void processFile( String path )
         throws ConsumerException
     {
@@ -129,6 +145,33 @@ public class NexusIndexerConsumer
         }
     }
 
+    @Override
+    public void processFile( String path, boolean executeOnEntireRepo )
+        throws Exception
+    {
+        if( executeOnEntireRepo )
+        {
+            processFile( path );
+        }
+        else
+        {
+            File artifactFile = new File( managedRepository, path );
+
+            // specify in indexing task that this is not a repo scan request!
+            ArtifactIndexingTask task =
+                new ArtifactIndexingTask( repository, artifactFile, ArtifactIndexingTask.Action.ADD, context, false );
+            try
+            {
+                log.debug( "Queueing indexing task + '" + task + "' to add or update the artifact in the index." );
+                scheduler.queueTask( task );
+            }
+            catch ( TaskQueueException e )
+            {
+                throw new ConsumerException( e.getMessage(), e );
+            }
+        }
+    }
+
     public void completeScan()
     {
         ArtifactIndexingTask task =
@@ -145,6 +188,17 @@ public class NexusIndexerConsumer
         context = null;
     }
 
+    @Override
+    public void completeScan( boolean executeOnEntireRepo )
+    {
+        if( executeOnEntireRepo )
+        {
+            completeScan();
+        }
+
+        // else, do nothing as the context will be closed when indexing task is executed if not a repo scan request!
+    }
+
     public List<String> getExcludes()
     {
         return Collections.emptyList();
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/resources/META-INF/spring-context.xml b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/resources/META-INF/spring-context.xml
index 542215ff2..8282c6cb2 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/resources/META-INF/spring-context.xml
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/main/resources/META-INF/spring-context.xml
@@ -8,4 +8,8 @@
         <constructor-arg ref="archivaConfiguration"/>
         <constructor-arg ref="fileTypes"/>
     </bean>
+
+    <bean id="logger" class="org.apache.maven.archiva.common.utils.Slf4JPlexusLogger">
+      <constructor-arg type="java.lang.Class"><value>org.sonatype.nexus.index.DefaultNexusIndexer</value></constructor-arg>
+    </bean>
 </beans>
\ No newline at end of file
diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-metadata-consumer/src/main/java/org/apache/archiva/consumers/metadata/ArchivaMetadataCreationConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-metadata-consumer/src/main/java/org/apache/archiva/consumers/metadata/ArchivaMetadataCreationConsumer.java
index 762f79d55..61e496f6a 100644
--- a/archiva-modules/archiva-base/archiva-consumers/archiva-metadata-consumer/src/main/java/org/apache/archiva/consumers/metadata/ArchivaMetadataCreationConsumer.java
+++ b/archiva-modules/archiva-base/archiva-consumers/archiva-metadata-consumer/src/main/java/org/apache/archiva/consumers/metadata/ArchivaMetadataCreationConsumer.java
@@ -127,6 +127,12 @@ public class ArchivaMetadataCreationConsumer
         this.whenGathered = whenGathered;
     }
 
+    public void beginScan( ManagedRepositoryConfiguration repository, Date whenGathered, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        beginScan( repository, whenGathered );
+    }
+
     public void processFile( String path )
         throws ConsumerException
     {
@@ -174,11 +180,22 @@ public class ArchivaMetadataCreationConsumer
         metadataRepository.updateProject( repoId, project );
     }
 
+    public void processFile( String path, boolean executeOnEntireRepo )
+        throws ConsumerException
+    {
+        processFile( path );
+    }
+
     public void completeScan()
     {
         /* do nothing */
     }
 
+    public void completeScan( boolean executeOnEntireRepo )
+    {
+        completeScan();
+    }
+
     public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
     {
         if ( ConfigurationNames.isRepositoryScanning( propertyName ) )