start refactoring merging remote indexes to have a cron job doing it

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1551122 13f79535-47bb-0310-9956-ffa450edef68

archiva-modules/archiva-base/archiva-configuration/src/main/mdo/configuration.mdo

@@ -856,6 +856,16 @@
           <defaultValue>30</defaultValue>
           <description>The time to live of the merged index of the repository group.</description>
         </field>
+        <field>
+          <name>mergedIndexCronExpression</name>
+          <version>1.0.0+</version>
+          <type>String</type>
+          <description>
+          When to run the index merging for this group.
+          No default value.
+          </description>
+          <defaultValue></defaultValue>
+        </field>
         <field>
           <name>repositories</name>
           <version>1.2.0+</version>

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java

@@ -80,8 +80,8 @@ public class DefaultIndexMerger
         StopWatch stopWatch = new StopWatch();
         stopWatch.reset();
         stopWatch.start();
-        File tempRepoFile = Files.createTempDir();
-        tempRepoFile.deleteOnExit();
+
+        File tempRepoFile = indexMergerRequest.getMergedIndexDirectory();
 
         String tempRepoId = tempRepoFile.getName();
 

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java

@@ -18,6 +18,7 @@ package org.apache.archiva.indexer.merger;
  * under the License.
  */
 
+import java.io.File;
 import java.util.Collection;
 
 /**
@@ -44,6 +45,8 @@ public class IndexMergerRequest
 
     private int mergedIndexTtl;
 
+    private File mergedIndexDirectory;
+
     public IndexMergerRequest( Collection<String> repositoriesIds, boolean packIndex, String groupId )
     {
         this.repositoriesIds = repositoriesIds;
@@ -54,8 +57,8 @@ public class IndexMergerRequest
     /**
      * @since 1.4-M4
      */
-    public IndexMergerRequest(Collection<String> repositoriesIds, boolean packIndex, String groupId,
-                              String mergedIndexPath, int mergedIndexTtl)
+    public IndexMergerRequest( Collection<String> repositoriesIds, boolean packIndex, String groupId,
+                               String mergedIndexPath, int mergedIndexTtl )
     {
         this.repositoriesIds = repositoriesIds;
         this.packIndex = packIndex;
@@ -104,14 +107,33 @@ public class IndexMergerRequest
         this.mergedIndexPath = mergedIndexPath;
     }
 
-    public int getMergedIndexTtl() {
+    public int getMergedIndexTtl()
+    {
         return mergedIndexTtl;
     }
 
-    public void setMergedIndexTtl(int mergedIndexTtl) {
+    public void setMergedIndexTtl( int mergedIndexTtl )
+    {
         this.mergedIndexTtl = mergedIndexTtl;
     }
 
+    public File getMergedIndexDirectory()
+    {
+        return mergedIndexDirectory;
+    }
+
+    public void setMergedIndexDirectory( File mergedIndexDirectory )
+    {
+        this.mergedIndexDirectory = mergedIndexDirectory;
+    }
+
+    public IndexMergerRequest mergedIndexDirectory( File mergedIndexDirectory )
+    {
+        this.mergedIndexDirectory = mergedIndexDirectory;
+        return this;
+    }
+
+
     @Override
     public String toString()
     {
@@ -120,7 +142,8 @@ public class IndexMergerRequest
         sb.append( ", packIndex=" ).append( packIndex );
         sb.append( ", groupId='" ).append( groupId ).append( '\'' );
         sb.append( ", mergedIndexPath='" ).append( mergedIndexPath ).append( '\'' );
-        sb.append( ", mergedIndexTtl='" ).append( mergedIndexTtl ).append( '\'' );
+        sb.append( ", mergedIndexTtl=" ).append( mergedIndexTtl );
+        sb.append( ", mergedIndexDirectory='" ).append( mergedIndexDirectory ).append( '\'' );
         sb.append( '}' );
         return sb.toString();
     }

archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/RepositoryGroup.java

@@ -51,6 +51,12 @@ public class RepositoryGroup
      */
     private int mergedIndexTtl = 30;
 
+    /**
+     * default model value is empty so none
+     * @since 2.0.0
+     */
+    private String mergedIndexCronExpression;
+
     public RepositoryGroup()
     {
         // no op
@@ -160,6 +166,22 @@ public class RepositoryGroup
         return this;
     }
 
+    public String getMergedIndexCronExpression()
+    {
+        return mergedIndexCronExpression;
+    }
+
+    public void setMergedIndexCronExpression( String mergedIndexCronExpression )
+    {
+        this.mergedIndexCronExpression = mergedIndexCronExpression;
+    }
+
+    public RepositoryGroup mergedIndexCronExpression( String mergedIndexCronExpression )
+    {
+        this.mergedIndexCronExpression = mergedIndexCronExpression;
+        return this;
+    }
+
     public boolean equals( Object other )
     {
         if ( this == other )
@@ -188,10 +210,12 @@ public class RepositoryGroup
     @Override
     public String toString()
     {
-        final StringBuilder sb = new StringBuilder();
-        sb.append( "RepositoryGroup" );
-        sb.append( "{id='" ).append( id ).append( '\'' );
+        final StringBuilder sb = new StringBuilder( "RepositoryGroup{" );
+        sb.append( "id='" ).append( id ).append( '\'' );
         sb.append( ", repositories=" ).append( repositories );
+        sb.append( ", mergedIndexPath='" ).append( mergedIndexPath ).append( '\'' );
+        sb.append( ", mergedIndexTtl=" ).append( mergedIndexTtl );
+        sb.append( ", mergedIndexCronExpression='" ).append( mergedIndexCronExpression ).append( '\'' );
         sb.append( '}' );
         return sb.toString();
     }

archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/group/RepositoryGroupAdmin.java

@@ -22,6 +22,7 @@ import org.apache.archiva.admin.model.AuditInformation;
 import org.apache.archiva.admin.model.RepositoryAdminException;
 import org.apache.archiva.admin.model.beans.RepositoryGroup;
 
+import java.io.File;
 import java.util.List;
 import java.util.Map;
 
@@ -73,4 +74,6 @@ public interface RepositoryGroupAdmin
      */
     Map<String, List<String>> getRepositoryToGroupMap()
         throws RepositoryAdminException;
+
+    File getMergedIndexDirectory( String repositoryGroupId );
 }

archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java

@@ -33,7 +33,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.PostConstruct;
 import javax.inject.Inject;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -58,6 +60,25 @@ public class DefaultRepositoryGroupAdmin
     @Inject
     private ManagedRepositoryAdmin managedRepositoryAdmin;
 
+    private File groupsDirectory;
+
+    @PostConstruct
+    public void initialize()
+    {
+        String appServerBase = getRegistry().getString( "appserver.base" );
+        groupsDirectory = new File( appServerBase + File.separatorChar + "groups" );
+        if ( !groupsDirectory.exists() )
+        {
+            groupsDirectory.mkdirs();
+        }
+    }
+
+    @Override
+    public File getMergedIndexDirectory( String repositoryGroupId )
+    {
+        return new File( groupsDirectory, repositoryGroupId );
+    }
+
     public List<RepositoryGroup> getRepositoriesGroups()
         throws RepositoryAdminException
     {
@@ -68,7 +89,8 @@ public class DefaultRepositoryGroupAdmin
         {
             repositoriesGroups.add( new RepositoryGroup( repositoryGroupConfiguration.getId(), new ArrayList<String>(
                 repositoryGroupConfiguration.getRepositories() ) ).mergedIndexPath(
-                repositoryGroupConfiguration.getMergedIndexPath() ).mergedIndexTtl( repositoryGroupConfiguration.getMergedIndexTtl() ) );
+                repositoryGroupConfiguration.getMergedIndexPath() ).mergedIndexTtl(
+                repositoryGroupConfiguration.getMergedIndexTtl() ) );
         }
 
         return repositoriesGroups;
@@ -285,7 +307,7 @@ public class DefaultRepositoryGroupAdmin
                 "Invalid character(s) found in identifier. Only the following characters are allowed: alphanumeric, '.', '-' and '_'" );
         }
 
-        if ( repositoryGroup.getMergedIndexTtl() <= 0)
+        if ( repositoryGroup.getMergedIndexTtl() <= 0 )
         {
             throw new RepositoryAdminException( "Merged Index TTL must be greater than 0." );
         }

archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java

@@ -19,6 +19,7 @@ package org.apache.archiva.webdav;
  * under the License.
  */
 
+import com.google.common.io.Files;
 import org.apache.archiva.admin.model.RepositoryAdminException;
 import org.apache.archiva.admin.model.beans.ManagedRepository;
 import org.apache.archiva.admin.model.beans.RemoteRepository;
@@ -37,6 +38,7 @@ import org.apache.archiva.configuration.RepositoryGroupConfiguration;
 import org.apache.archiva.indexer.merger.IndexMerger;
 import org.apache.archiva.indexer.merger.IndexMergerException;
 import org.apache.archiva.indexer.merger.IndexMergerRequest;
+import org.apache.archiva.indexer.merger.MergedRemoteIndexesTaskJob;
 import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
 import org.apache.archiva.indexer.search.RepositorySearch;
 import org.apache.archiva.maven2.metadata.MavenMetadataReader;
@@ -115,7 +117,7 @@ import java.util.Set;
 /**
  *
  */
-@Service( "davResourceFactory#archiva" )
+@Service("davResourceFactory#archiva")
 public class ArchivaDavResourceFactory
     implements DavResourceFactory, Auditable
 {
@@ -134,7 +136,7 @@ public class ArchivaDavResourceFactory
     private RepositoryRequest repositoryRequest;
 
     @Inject
-    @Named( value = "repositoryProxyConnectors#default" )
+    @Named(value = "repositoryProxyConnectors#default")
     private RepositoryProxyConnectors connectors;
 
     @Inject
@@ -149,7 +151,7 @@ public class ArchivaDavResourceFactory
     private ServletAuthenticator servletAuth;
 
     @Inject
-    @Named( value = "httpAuthenticator#basic" )
+    @Named(value = "httpAuthenticator#basic")
     private HttpAuthenticator httpAuth;
 
     @Inject
@@ -176,11 +178,11 @@ public class ArchivaDavResourceFactory
     private Digester digestMd5;
 
     @Inject
-    @Named( value = "archivaTaskScheduler#repository" )
+    @Named(value = "archivaTaskScheduler#repository")
     private RepositoryArchivaTaskScheduler scheduler;
 
     @Inject
-    @Named(value= "fileLockManager#default")
+    @Named(value = "fileLockManager#default")
     private FileLockManager fileLockManager;
 
     private ApplicationContext applicationContext;
@@ -345,12 +347,13 @@ public class ArchivaDavResourceFactory
                     if ( metadataChecksum.exists() )
                     {
                         LogicalResource logicalResource =
-                            new LogicalResource (getLogicalResource( archivaLocator, null, false ) );
+                            new LogicalResource( getLogicalResource( archivaLocator, null, false ) );
 
                         resource =
                             new ArchivaDavResource( metadataChecksum.getAbsolutePath(), logicalResource.getPath(), null,
                                                     request.getRemoteAddr(), activePrincipal, request.getDavSession(),
-                                                    archivaLocator, this, mimeTypes, auditListeners, scheduler, fileLockManager );
+                                                    archivaLocator, this, mimeTypes, auditListeners, scheduler,
+                                                    fileLockManager );
                     }
                 }
                 else
@@ -383,7 +386,8 @@ public class ArchivaDavResourceFactory
                         {
                             File resourceFile = writeMergedMetadataToFile( mergedMetadata, filePath );
 
-                            LogicalResource logicalResource = new LogicalResource( getLogicalResource( archivaLocator, null, false ) );
+                            LogicalResource logicalResource =
+                                new LogicalResource( getLogicalResource( archivaLocator, null, false ) );
 
                             resource =
                                 new ArchivaDavResource( resourceFile.getAbsolutePath(), logicalResource.getPath(), null,
@@ -404,7 +408,8 @@ public class ArchivaDavResourceFactory
                         catch ( DigesterException de )
                         {
                             throw new DavException( HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
-                                                    "Error occurred while generating checksum files." + de.getMessage() );
+                                                    "Error occurred while generating checksum files."
+                                                        + de.getMessage() );
                         }
                     }
                 }
@@ -445,7 +450,8 @@ public class ArchivaDavResourceFactory
             File resourceFile = new File( temporaryIndexDirectory, requestedFileName );
             resource = new ArchivaDavResource( resourceFile.getAbsolutePath(), requestedFileName, null,
                                                request.getRemoteAddr(), activePrincipal, request.getDavSession(),
-                                               archivaLocator, this, mimeTypes, auditListeners, scheduler, fileLockManager );
+                                               archivaLocator, this, mimeTypes, auditListeners, scheduler,
+                                               fileLockManager );
 
         }
         else
@@ -523,32 +529,39 @@ public class ArchivaDavResourceFactory
     {
         // FIXME remove this hack
         // but currently managedRepository can be null in case of group
-        String layout = managedRepository == null ? new ManagedRepository( ).getLayout() : managedRepository.getLayout();
+        String layout = managedRepository == null ? new ManagedRepository().getLayout() : managedRepository.getLayout();
         RepositoryStorage repositoryStorage =
             this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class );
         String path = repositoryStorage.getFilePath(
-            useOrigResourcePath ? archivaLocator.getOrigResourcePath() : archivaLocator.getResourcePath(), managedRepository );
+            useOrigResourcePath ? archivaLocator.getOrigResourcePath() : archivaLocator.getResourcePath(),
+            managedRepository );
         log.debug( "found path {} for resourcePath: '{}' with managedRepo '{}' and layout '{}'", path,
-                   archivaLocator.getResourcePath(), managedRepository == null ? "null" : managedRepository.getId(), layout );
+                   archivaLocator.getResourcePath(), managedRepository == null ? "null" : managedRepository.getId(),
+                   layout );
         return path;
     }
 
-    private String evaluatePathWithVersion( ArchivaDavResourceLocator archivaLocator, ManagedRepositoryContent managedRepositoryContent, String contextPath )
+    private String evaluatePathWithVersion( ArchivaDavResourceLocator archivaLocator,
+                                            ManagedRepositoryContent managedRepositoryContent, String contextPath )
         throws DavException
     {
-        String layout = managedRepositoryContent.getRepository() == null ? new ManagedRepository( ).getLayout() : managedRepositoryContent.getRepository().getLayout();
+        String layout = managedRepositoryContent.getRepository() == null
+            ? new ManagedRepository().getLayout()
+            : managedRepositoryContent.getRepository().getLayout();
         RepositoryStorage repositoryStorage =
             this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class );
         try
         {
-            return repositoryStorage.getFilePathWithVersion( archivaLocator.getResourcePath(), managedRepositoryContent );
+            return repositoryStorage.getFilePathWithVersion( archivaLocator.getResourcePath(),
+                                                             managedRepositoryContent );
         }
         catch ( RelocationException e )
         {
             String path = e.getPath();
             log.debug( "Relocation to {}", path );
 
-            throw new BrowserRedirectException(contextPath + ( StringUtils.startsWith( path, "/" ) ? "": "/" ) + path, e.getRelocationType() );
+            throw new BrowserRedirectException( contextPath + ( StringUtils.startsWith( path, "/" ) ? "" : "/" ) + path,
+                                                e.getRelocationType() );
         }
         catch ( XMLException e )
         {
@@ -566,7 +579,7 @@ public class ArchivaDavResourceFactory
         if ( isAuthorized( request, managedRepositoryContent.getId() ) )
         {
             // Maven Centric part ask evaluation if -SNAPSHOT
-            String path = evaluatePathWithVersion(archivaLocator, managedRepositoryContent, request.getContextPath());
+            String path = evaluatePathWithVersion( archivaLocator, managedRepositoryContent, request.getContextPath() );
             if ( path.startsWith( "/" ) )
             {
                 path = path.substring( 1 );
@@ -1272,12 +1285,14 @@ public class ArchivaDavResourceFactory
             }
 
             Set<String> authzRepos = new HashSet<String>();
+
+            String permission = WebdavMethodUtil.getMethodPermission( request.getMethod() );
+
             for ( String repository : repositories )
             {
                 try
                 {
-                    if ( servletAuth.isAuthorized( activePrincipal, repository,
-                                                   WebdavMethodUtil.getMethodPermission( request.getMethod() ) ) )
+                    if ( servletAuth.isAuthorized( activePrincipal, repository, permission ) )
                     {
                         authzRepos.add( repository );
                         authzRepos.addAll( this.repositorySearch.getRemoteIndexingContextIds( repository ) );
@@ -1293,10 +1308,22 @@ public class ArchivaDavResourceFactory
             }
             log.info( "generate temporary merged index for repository group '{}' for repositories '{}'",
                       repositoryGroupConfiguration.getId(), authzRepos );
-            IndexingContext indexingContext = indexMerger.buildMergedIndex(
-                new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(),
-                                        repositoryGroupConfiguration.getMergedIndexPath(),
-                                        repositoryGroupConfiguration.getMergedIndexTtl() ) );
+
+            File tempRepoFile = Files.createTempDir();
+            tempRepoFile.deleteOnExit();
+
+            IndexMergerRequest indexMergerRequest = new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(),
+                                    repositoryGroupConfiguration.getMergedIndexPath(),
+                                    repositoryGroupConfiguration.getMergedIndexTtl() ).mergedIndexDirectory(
+                tempRepoFile );
+
+            MergedRemoteIndexesTaskJob job = new MergedRemoteIndexesTaskJob();
+
+            MergedRemoteIndexesTaskJob.MergedRemoteIndexesTaskRequest taskRequest =
+                new MergedRemoteIndexesTaskJob.MergedRemoteIndexesTaskRequest(indexMergerRequest, indexMerger);
+
+            IndexingContext indexingContext = job.execute( taskRequest ).getIndexingContext();
+
             File mergedRepoDir = indexingContext.getIndexDirectoryFile();
             TemporaryGroupIndex temporaryGroupIndex =
                 new TemporaryGroupIndex( mergedRepoDir, indexingContext.getId(), repositoryGroupConfiguration.getId(),