[MRM-815] aggregate indices for repository groups.

delete temporary on session end with a session listener
periodical taks to cleanup too old temp group index.

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1198011 13f79535-47bb-0310-9956-ffa450edef68

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java

@@ -23,7 +23,6 @@ import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
 import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
 import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
-import org.apache.commons.io.FileUtils;
 import org.apache.maven.index.NexusIndexer;
 import org.apache.maven.index.context.IndexingContext;
 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
@@ -31,14 +30,13 @@ import org.apache.maven.index.packer.IndexPacker;
 import org.apache.maven.index.packer.IndexPackingRequest;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
 import javax.inject.Inject;
 import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Date;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 
@@ -62,7 +60,7 @@ public class DefaultIndexMerger
 
     private IndexPacker indexPacker;
 
-    private List<TemporaryIndex> temporaryIndexes = new CopyOnWriteArrayList<TemporaryIndex>();
+    private List<TemporaryGroupIndex> temporaryGroupIndexes = new CopyOnWriteArrayList<TemporaryGroupIndex>();
 
     @Inject
     public DefaultIndexMerger( PlexusSisuBridge plexusSisuBridge, MavenIndexerUtils mavenIndexerUtils )
@@ -73,7 +71,7 @@ public class DefaultIndexMerger
         indexPacker = plexusSisuBridge.lookup( IndexPacker.class, "default" );
     }
 
-    public File buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
+    public IndexingContext buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
         throws IndexMergerException
     {
         File tempRepoFile = Files.createTempDir();
@@ -104,8 +102,8 @@ public class DefaultIndexMerger
                 IndexPackingRequest request = new IndexPackingRequest( indexingContext, indexLocation );
                 indexPacker.packIndex( request );
             }
-            temporaryIndexes.add( new TemporaryIndex( tempRepoFile, tempRepoId ) );
-            return indexingContext.getIndexDirectoryFile();
+            temporaryGroupIndexes.add( new TemporaryGroupIndex( tempRepoFile, tempRepoId ) );
+            return indexingContext;
         }
         catch ( IOException e )
         {
@@ -117,70 +115,30 @@ public class DefaultIndexMerger
         }
     }
 
-
-    @Scheduled( fixedDelay = 900000 )
-    public void cleanTemporaryIndex()
+    @Async
+    public void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex )
     {
-        for ( TemporaryIndex temporaryIndex : temporaryIndexes )
+        if ( temporaryGroupIndex == null )
         {
-            // cleanup files older than 30 minutes
-            if ( new Date().getTime() - temporaryIndex.creationTime > 1800000 )
-            {
-                try
-                {
-                    IndexingContext context = indexer.getIndexingContexts().get( temporaryIndex.indexId );
-                    if ( context != null )
-                    {
-                        indexer.removeIndexingContext( context, true );
-                    }
-                    else
-                    {
-                        FileUtils.deleteDirectory( temporaryIndex.directory );
-                    }
-                    temporaryIndexes.remove( temporaryIndex );
-                    log.debug( "remove directory {}", temporaryIndex.directory );
-                }
-                catch ( IOException e )
-                {
-                    log.warn( "failed to remove directory:" + temporaryIndex.directory, e );
-                }
-            }
-            temporaryIndexes.remove( temporaryIndex );
+            return;
         }
-    }
-
-    private static class TemporaryIndex
-    {
-        private long creationTime = new Date().getTime();
 
-        private File directory;
-
-        private String indexId;
-
-        TemporaryIndex( File directory, String indexId )
+        try
         {
-            this.directory = directory;
-            this.indexId = indexId;
+            IndexingContext indexingContext = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() );
+            if ( indexingContext != null )
+            {
+                indexer.removeIndexingContext( indexingContext, true );
+            }
         }
-
-        @Override
-        public int hashCode()
+        catch ( IOException e )
         {
-            return Long.toString( creationTime ).hashCode();
+            log.warn( "fail to delete temporary group index {}", temporaryGroupIndex.getIndexId(), e );
         }
+    }
 
-        @Override
-        public boolean equals( Object o )
-        {
-            if ( this == o )
-            {
-                return true;
-            }
-            if ( !( o instanceof TemporaryIndex ) )
-            {
-                return false;
-            }
-            return this.creationTime == ( (TemporaryIndex) o ).creationTime;
-        }
+    public Collection<TemporaryGroupIndex> getTemporaryGroupIndexes()
+    {
+        return this.temporaryGroupIndexes;
     }
 }

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java

@@ -18,6 +18,8 @@ package org.apache.archiva.indexer.merger;
  * under the License.
  */
 
+import org.apache.maven.index.context.IndexingContext;
+
 import java.io.File;
 import java.util.Collection;
 
@@ -27,12 +29,21 @@ import java.util.Collection;
  */
 public interface IndexMerger
 {
+    /**
+     * default tmp created group index ttl in minutes
+     */
+    static final int DEFAULT_GROUP_INDEX_TTL = 1;
+
     /**
      * @param repositoriesIds repositories Ids to merge content
-     * @param packIndex will generate a downloadable index
+     * @param packIndex       will generate a downloadable index
      * @return a temporary directory with a merge index (directory marked deleteOnExit)
      * @throws IndexMergerException
      */
-    File buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
+    IndexingContext buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
         throws IndexMergerException;
+
+    void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex );
+
+    Collection<TemporaryGroupIndex> getTemporaryGroupIndexes();
 }

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java

@@ -0,0 +1,95 @@
+package org.apache.archiva.indexer.merger;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * @author Olivier Lamy
+ */
+public class TemporaryGroupIndex
+    implements Serializable
+{
+    private long creationTime = new Date().getTime();
+
+    private File directory;
+
+    private String indexId;
+
+    public TemporaryGroupIndex( File directory, String indexId )
+    {
+        this.directory = directory;
+        this.indexId = indexId;
+    }
+
+    public long getCreationTime()
+    {
+        return creationTime;
+    }
+
+    public TemporaryGroupIndex setCreationTime( long creationTime )
+    {
+        this.creationTime = creationTime;
+        return this;
+    }
+
+    public File getDirectory()
+    {
+        return directory;
+    }
+
+    public TemporaryGroupIndex setDirectory( File directory )
+    {
+        this.directory = directory;
+        return this;
+    }
+
+    public String getIndexId()
+    {
+        return indexId;
+    }
+
+    public TemporaryGroupIndex setIndexId( String indexId )
+    {
+        this.indexId = indexId;
+        return this;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Long.toString( creationTime ).hashCode();
+    }
+
+    @Override
+    public boolean equals( Object o )
+    {
+        if ( this == o )
+        {
+            return true;
+        }
+        if ( !( o instanceof TemporaryGroupIndex ) )
+        {
+            return false;
+        }
+        return this.creationTime == ( (TemporaryGroupIndex) o ).creationTime;
+    }
+}

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java

@@ -0,0 +1,85 @@
+package org.apache.archiva.indexer.merger;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
+import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
+import org.apache.maven.index.NexusIndexer;
+import org.apache.maven.index.context.IndexingContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * @author Olivier Lamy
+ * @since 1.4-M2
+ */
+@Service
+public class TemporaryGroupIndexCleaner
+{
+    private Logger log = LoggerFactory.getLogger( getClass() );
+
+    @Inject
+    private IndexMerger indexMerger;
+
+    private NexusIndexer indexer;
+
+    @Inject
+    public TemporaryGroupIndexCleaner( PlexusSisuBridge plexusSisuBridge )
+        throws PlexusSisuBridgeException
+    {
+        indexer = plexusSisuBridge.lookup( NexusIndexer.class );
+    }
+
+    // 900000
+    @Scheduled( fixedDelay = 900000 )
+    public void cleanTemporaryIndex()
+    {
+        for ( TemporaryGroupIndex temporaryGroupIndex : indexMerger.getTemporaryGroupIndexes() )
+        {
+            // cleanup files older than 60 minutes 3600000
+            if ( new Date().getTime() - temporaryGroupIndex.getCreationTime() > 3600000 )
+            {
+                try
+                {
+                    IndexingContext context = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() );
+                    if ( context != null )
+                    {
+                        indexer.removeIndexingContext( context, true );
+                    }
+                    else
+                    {
+                        indexMerger.cleanTemporaryGroupIndex( temporaryGroupIndex );
+                    }
+                    indexMerger.getTemporaryGroupIndexes().remove( temporaryGroupIndex );
+                    log.debug( "remove directory {}", temporaryGroupIndex.getDirectory() );
+                }
+                catch ( IOException e )
+                {
+                    log.warn( "failed to remove directory:" + temporaryGroupIndex.getDirectory(), e );
+                }
+            }
+        }
+    }
+}

archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/web.xml

@@ -89,6 +89,11 @@
     <listener-class>net.sf.ehcache.constructs.web.ShutdownListener</listener-class>
   </listener>
 
+  <!-- to cleanup temporary group index created during a session -->
+  <listener>
+    <listener-class>org.apache.archiva.webdav.util.TemporaryGroupIndexSessionCleaner</listener-class>
+  </listener>
+
 	<context-param>
 		<param-name>contextConfigLocation</param-name>
 		<param-value>

archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java

@@ -31,6 +31,7 @@ import org.apache.archiva.configuration.ArchivaConfiguration;
 import org.apache.archiva.configuration.RepositoryGroupConfiguration;
 import org.apache.archiva.indexer.merger.IndexMerger;
 import org.apache.archiva.indexer.merger.IndexMergerException;
+import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
 import org.apache.archiva.indexer.search.RepositorySearch;
 import org.apache.archiva.model.ArchivaRepositoryMetadata;
 import org.apache.archiva.model.ArtifactReference;
@@ -52,6 +53,7 @@ import org.apache.archiva.scheduler.repository.RepositoryArchivaTaskScheduler;
 import org.apache.archiva.security.ServletAuthenticator;
 import org.apache.archiva.webdav.util.MimeTypes;
 import org.apache.archiva.webdav.util.RepositoryPathUtil;
+import org.apache.archiva.webdav.util.TemporaryGroupIndexSessionCleaner;
 import org.apache.archiva.webdav.util.WebdavMethodUtil;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
@@ -65,6 +67,7 @@ import org.apache.jackrabbit.webdav.DavServletResponse;
 import org.apache.jackrabbit.webdav.DavSession;
 import org.apache.jackrabbit.webdav.lock.LockManager;
 import org.apache.jackrabbit.webdav.lock.SimpleLockManager;
+import org.apache.maven.index.context.IndexingContext;
 import org.apache.maven.model.DistributionManagement;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.Relocation;
@@ -98,6 +101,7 @@ import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -1198,16 +1202,29 @@ public class ArchivaDavResourceFactory
         try
         {
             HttpSession session = request.getSession();
-            Map<String, File> testValue = (Map<String, File>) session.getAttribute( "TMP_GROUP_INDEXES" );
-            if ( testValue == null )
+
+            Map<String, TemporaryGroupIndex> temporaryGroupIndexMap =
+                (Map<String, TemporaryGroupIndex>) session.getAttribute(
+                    TemporaryGroupIndexSessionCleaner.TEMPORARY_INDEX_SESSION_KEY );
+            if ( temporaryGroupIndexMap == null )
             {
-                testValue = new HashMap<String, File>();
+                temporaryGroupIndexMap = new HashMap<String, TemporaryGroupIndex>();
             }
 
-            File tmp = testValue.get( groupId );
-            if ( tmp != null )
+            TemporaryGroupIndex tmp = temporaryGroupIndexMap.get( groupId );
+
+            if ( tmp != null && tmp.getDirectory() != null && tmp.getDirectory().exists() )
             {
-                return tmp;
+                if ( System.currentTimeMillis() - tmp.getCreationTime() > ( IndexMerger.DEFAULT_GROUP_INDEX_TTL * 60
+                    * 1000 ) )
+                {
+                    log.debug( "tmp group index is too old so delete it" );
+                    indexMerger.cleanTemporaryGroupIndex( tmp );
+                }
+                else
+                {
+                    return tmp.getDirectory();
+                }
             }
 
             Set<String> authzRepos = new HashSet<String>();
@@ -1232,10 +1249,14 @@ public class ArchivaDavResourceFactory
                     }
                 }
             }
-
-            File mergedRepoDir = indexMerger.buildMergedIndex( authzRepos, true );
-            testValue.put( groupId, mergedRepoDir );
-            session.setAttribute( "TMP_GROUP_INDEXES", testValue );
+            IndexingContext indexingContext = indexMerger.buildMergedIndex( authzRepos, true );
+            File mergedRepoDir = indexingContext.getIndexDirectoryFile();
+            TemporaryGroupIndex temporaryGroupIndex =
+                new TemporaryGroupIndex( mergedRepoDir, indexingContext.getId() ).setCreationTime(
+                    new Date().getTime() );
+            temporaryGroupIndexMap.put( groupId, temporaryGroupIndex );
+            session.setAttribute( TemporaryGroupIndexSessionCleaner.TEMPORARY_INDEX_SESSION_KEY,
+                                  temporaryGroupIndexMap );
             return mergedRepoDir;
         }
         catch ( RepositoryAdminException e )

archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/util/TemporaryGroupIndexSessionCleaner.java

@@ -0,0 +1,74 @@
+package org.apache.archiva.webdav.util;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.archiva.indexer.merger.IndexMerger;
+import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.web.context.WebApplicationContext;
+import org.springframework.web.context.support.WebApplicationContextUtils;
+
+import javax.servlet.http.HttpSessionEvent;
+import javax.servlet.http.HttpSessionListener;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * this http session listener will delete repository group index requested by a user
+ * at this end of the http session
+ *
+ * @author Olivier Lamy
+ * @since 1.4-M2
+ */
+public class TemporaryGroupIndexSessionCleaner
+    implements HttpSessionListener
+{
+
+    private Logger log = LoggerFactory.getLogger( getClass() );
+
+    private IndexMerger indexMerger;
+
+    public static final String TEMPORARY_INDEX_SESSION_KEY = TemporaryGroupIndexSessionCleaner.class.getName();
+
+    public void sessionCreated( HttpSessionEvent httpSessionEvent )
+    {
+        // ensure the map is here to avoid NPE
+        httpSessionEvent.getSession().setAttribute( TEMPORARY_INDEX_SESSION_KEY,
+                                                    new HashMap<String, TemporaryGroupIndex>() );
+        WebApplicationContext webApplicationContext = WebApplicationContextUtils.getRequiredWebApplicationContext(
+            httpSessionEvent.getSession().getServletContext() );
+        indexMerger = webApplicationContext.getBean( IndexMerger.class );
+    }
+
+    public void sessionDestroyed( HttpSessionEvent httpSessionEvent )
+    {
+        Map<String, TemporaryGroupIndex> tempFilesPerKey =
+            (Map<String, TemporaryGroupIndex>) httpSessionEvent.getSession().getAttribute(
+                TEMPORARY_INDEX_SESSION_KEY );
+
+        for ( TemporaryGroupIndex temporaryGroupIndex : tempFilesPerKey.values() )
+        {
+            log.info( "cleanup temporaryGroupIndex {} directory {}", temporaryGroupIndex.getIndexId(),
+                      temporaryGroupIndex.getDirectory().getAbsolutePath() );
+            indexMerger.cleanTemporaryGroupIndex( temporaryGroupIndex );
+        }
+    }
+}
+}