[MRM-815] aggregate indices for repository groups.

delete temporary on session end with a session listener
periodical taks to cleanup too old temp group index.

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1198011 13f79535-47bb-0310-9956-ffa450edef68

4 files changed, 214 insertions, 65 deletions

diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java
index 9c0613490..674f261fb 100644
--- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java
+++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java
@@ -23,7 +23,6 @@ import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
 import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
 import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
-import org.apache.commons.io.FileUtils;
 import org.apache.maven.index.NexusIndexer;
 import org.apache.maven.index.context.IndexingContext;
 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
@@ -31,14 +30,13 @@ import org.apache.maven.index.packer.IndexPacker;
 import org.apache.maven.index.packer.IndexPackingRequest;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
 import javax.inject.Inject;
 import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Date;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 
@@ -62,7 +60,7 @@ public class DefaultIndexMerger
 
     private IndexPacker indexPacker;
 
-    private List<TemporaryIndex> temporaryIndexes = new CopyOnWriteArrayList<TemporaryIndex>();
+    private List<TemporaryGroupIndex> temporaryGroupIndexes = new CopyOnWriteArrayList<TemporaryGroupIndex>();
 
     @Inject
     public DefaultIndexMerger( PlexusSisuBridge plexusSisuBridge, MavenIndexerUtils mavenIndexerUtils )
@@ -73,7 +71,7 @@ public class DefaultIndexMerger
         indexPacker = plexusSisuBridge.lookup( IndexPacker.class, "default" );
     }
 
-    public File buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
+    public IndexingContext buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
         throws IndexMergerException
     {
         File tempRepoFile = Files.createTempDir();
@@ -104,8 +102,8 @@ public class DefaultIndexMerger
                 IndexPackingRequest request = new IndexPackingRequest( indexingContext, indexLocation );
                 indexPacker.packIndex( request );
             }
-            temporaryIndexes.add( new TemporaryIndex( tempRepoFile, tempRepoId ) );
-            return indexingContext.getIndexDirectoryFile();
+            temporaryGroupIndexes.add( new TemporaryGroupIndex( tempRepoFile, tempRepoId ) );
+            return indexingContext;
         }
         catch ( IOException e )
         {
@@ -117,70 +115,30 @@ public class DefaultIndexMerger
         }
     }
 
-
-    @Scheduled( fixedDelay = 900000 )
-    public void cleanTemporaryIndex()
+    @Async
+    public void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex )
     {
-        for ( TemporaryIndex temporaryIndex : temporaryIndexes )
+        if ( temporaryGroupIndex == null )
         {
-            // cleanup files older than 30 minutes
-            if ( new Date().getTime() - temporaryIndex.creationTime > 1800000 )
-            {
-                try
-                {
-                    IndexingContext context = indexer.getIndexingContexts().get( temporaryIndex.indexId );
-                    if ( context != null )
-                    {
-                        indexer.removeIndexingContext( context, true );
-                    }
-                    else
-                    {
-                        FileUtils.deleteDirectory( temporaryIndex.directory );
-                    }
-                    temporaryIndexes.remove( temporaryIndex );
-                    log.debug( "remove directory {}", temporaryIndex.directory );
-                }
-                catch ( IOException e )
-                {
-                    log.warn( "failed to remove directory:" + temporaryIndex.directory, e );
-                }
-            }
-            temporaryIndexes.remove( temporaryIndex );
+            return;
         }
-    }
-
-    private static class TemporaryIndex
-    {
-        private long creationTime = new Date().getTime();
 
-        private File directory;
-
-        private String indexId;
-
-        TemporaryIndex( File directory, String indexId )
+        try
         {
-            this.directory = directory;
-            this.indexId = indexId;
+            IndexingContext indexingContext = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() );
+            if ( indexingContext != null )
+            {
+                indexer.removeIndexingContext( indexingContext, true );
+            }
         }
-
-        @Override
-        public int hashCode()
+        catch ( IOException e )
         {
-            return Long.toString( creationTime ).hashCode();
+            log.warn( "fail to delete temporary group index {}", temporaryGroupIndex.getIndexId(), e );
         }
+    }
 
-        @Override
-        public boolean equals( Object o )
-        {
-            if ( this == o )
-            {
-                return true;
-            }
-            if ( !( o instanceof TemporaryIndex ) )
-            {
-                return false;
-            }
-            return this.creationTime == ( (TemporaryIndex) o ).creationTime;
-        }
+    public Collection<TemporaryGroupIndex> getTemporaryGroupIndexes()
+    {
+        return this.temporaryGroupIndexes;
     }
 }
diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java
index 89e2c7d6f..17a1f21f6 100644
--- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java
+++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java
@@ -18,6 +18,8 @@ package org.apache.archiva.indexer.merger;
  * under the License.
  */
 
+import org.apache.maven.index.context.IndexingContext;
+
 import java.io.File;
 import java.util.Collection;
 
@@ -28,11 +30,20 @@ import java.util.Collection;
 public interface IndexMerger
 {
     /**
+     * default tmp created group index ttl in minutes
+     */
+    static final int DEFAULT_GROUP_INDEX_TTL = 1;
+
+    /**
      * @param repositoriesIds repositories Ids to merge content
-     * @param packIndex will generate a downloadable index
+     * @param packIndex       will generate a downloadable index
      * @return a temporary directory with a merge index (directory marked deleteOnExit)
      * @throws IndexMergerException
      */
-    File buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
+    IndexingContext buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
         throws IndexMergerException;
+
+    void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex );
+
+    Collection<TemporaryGroupIndex> getTemporaryGroupIndexes();
 }
diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java
new file mode 100644
index 000000000..489ac1594
--- /dev/null
+++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java
@@ -0,0 +1,95 @@
+package org.apache.archiva.indexer.merger;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * @author Olivier Lamy
+ */
+public class TemporaryGroupIndex
+    implements Serializable
+{
+    private long creationTime = new Date().getTime();
+
+    private File directory;
+
+    private String indexId;
+
+    public TemporaryGroupIndex( File directory, String indexId )
+    {
+        this.directory = directory;
+        this.indexId = indexId;
+    }
+
+    public long getCreationTime()
+    {
+        return creationTime;
+    }
+
+    public TemporaryGroupIndex setCreationTime( long creationTime )
+    {
+        this.creationTime = creationTime;
+        return this;
+    }
+
+    public File getDirectory()
+    {
+        return directory;
+    }
+
+    public TemporaryGroupIndex setDirectory( File directory )
+    {
+        this.directory = directory;
+        return this;
+    }
+
+    public String getIndexId()
+    {
+        return indexId;
+    }
+
+    public TemporaryGroupIndex setIndexId( String indexId )
+    {
+        this.indexId = indexId;
+        return this;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Long.toString( creationTime ).hashCode();
+    }
+
+    @Override
+    public boolean equals( Object o )
+    {
+        if ( this == o )
+        {
+            return true;
+        }
+        if ( !( o instanceof TemporaryGroupIndex ) )
+        {
+            return false;
+        }
+        return this.creationTime == ( (TemporaryGroupIndex) o ).creationTime;
+    }
+}
diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java
new file mode 100644
index 000000000..0845db6b1
--- /dev/null
+++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java
@@ -0,0 +1,85 @@
+package org.apache.archiva.indexer.merger;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
+import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
+import org.apache.maven.index.NexusIndexer;
+import org.apache.maven.index.context.IndexingContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * @author Olivier Lamy
+ * @since 1.4-M2
+ */
+@Service
+public class TemporaryGroupIndexCleaner
+{
+    private Logger log = LoggerFactory.getLogger( getClass() );
+
+    @Inject
+    private IndexMerger indexMerger;
+
+    private NexusIndexer indexer;
+
+    @Inject
+    public TemporaryGroupIndexCleaner( PlexusSisuBridge plexusSisuBridge )
+        throws PlexusSisuBridgeException
+    {
+        indexer = plexusSisuBridge.lookup( NexusIndexer.class );
+    }
+
+    // 900000
+    @Scheduled( fixedDelay = 900000 )
+    public void cleanTemporaryIndex()
+    {
+        for ( TemporaryGroupIndex temporaryGroupIndex : indexMerger.getTemporaryGroupIndexes() )
+        {
+            // cleanup files older than 60 minutes 3600000
+            if ( new Date().getTime() - temporaryGroupIndex.getCreationTime() > 3600000 )
+            {
+                try
+                {
+                    IndexingContext context = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() );
+                    if ( context != null )
+                    {
+                        indexer.removeIndexingContext( context, true );
+                    }
+                    else
+                    {
+                        indexMerger.cleanTemporaryGroupIndex( temporaryGroupIndex );
+                    }
+                    indexMerger.getTemporaryGroupIndexes().remove( temporaryGroupIndex );
+                    log.debug( "remove directory {}", temporaryGroupIndex.getDirectory() );
+                }
+                catch ( IOException e )
+                {
+                    log.warn( "failed to remove directory:" + temporaryGroupIndex.getDirectory(), e );
+                }
+            }
+        }
+    }
+}