From: Olivier Lamy Date: Mon, 16 Dec 2013 06:53:27 +0000 (+0000) Subject: start refactoring merging remote indexes to have a cron job doing it X-Git-Tag: archiva-2.0.0-RC1~104 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=28898793f51c03b1259d3a1b1f1612f450cf634c;p=archiva.git start refactoring merging remote indexes to have a cron job doing it git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1551122 13f79535-47bb-0310-9956-ffa450edef68 --- diff --git a/archiva-modules/archiva-base/archiva-configuration/src/main/mdo/configuration.mdo b/archiva-modules/archiva-base/archiva-configuration/src/main/mdo/configuration.mdo index c64f8f8e6..df3f41f89 100644 --- a/archiva-modules/archiva-base/archiva-configuration/src/main/mdo/configuration.mdo +++ b/archiva-modules/archiva-base/archiva-configuration/src/main/mdo/configuration.mdo @@ -856,6 +856,16 @@ 30 The time to live of the merged index of the repository group. + + mergedIndexCronExpression + 1.0.0+ + String + + When to run the index merging for this group. + No default value. + + + repositories 1.2.0+ diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java index beccab06e..4a939cb49 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java @@ -80,8 +80,8 @@ public class DefaultIndexMerger StopWatch stopWatch = new StopWatch(); stopWatch.reset(); stopWatch.start(); - File tempRepoFile = Files.createTempDir(); - tempRepoFile.deleteOnExit(); + + File tempRepoFile = indexMergerRequest.getMergedIndexDirectory(); String tempRepoId = tempRepoFile.getName(); diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java index a7a3b656f..e99e3d562 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java @@ -18,6 +18,7 @@ package org.apache.archiva.indexer.merger; * under the License. */ +import java.io.File; import java.util.Collection; /** @@ -44,6 +45,8 @@ public class IndexMergerRequest private int mergedIndexTtl; + private File mergedIndexDirectory; + public IndexMergerRequest( Collection repositoriesIds, boolean packIndex, String groupId ) { this.repositoriesIds = repositoriesIds; @@ -54,8 +57,8 @@ public class IndexMergerRequest /** * @since 1.4-M4 */ - public IndexMergerRequest(Collection repositoriesIds, boolean packIndex, String groupId, - String mergedIndexPath, int mergedIndexTtl) + public IndexMergerRequest( Collection repositoriesIds, boolean packIndex, String groupId, + String mergedIndexPath, int mergedIndexTtl ) { this.repositoriesIds = repositoriesIds; this.packIndex = packIndex; @@ -104,14 +107,33 @@ public class IndexMergerRequest this.mergedIndexPath = mergedIndexPath; } - public int getMergedIndexTtl() { + public int getMergedIndexTtl() + { return mergedIndexTtl; } - public void setMergedIndexTtl(int mergedIndexTtl) { + public void setMergedIndexTtl( int mergedIndexTtl ) + { this.mergedIndexTtl = mergedIndexTtl; } + public File getMergedIndexDirectory() + { + return mergedIndexDirectory; + } + + public void setMergedIndexDirectory( File mergedIndexDirectory ) + { + this.mergedIndexDirectory = mergedIndexDirectory; + } + + public IndexMergerRequest mergedIndexDirectory( File mergedIndexDirectory ) + { + this.mergedIndexDirectory = mergedIndexDirectory; + return this; + } + + @Override public String toString() { @@ -120,7 +142,8 @@ public class IndexMergerRequest sb.append( ", packIndex=" ).append( packIndex ); sb.append( ", groupId='" ).append( groupId ).append( '\'' ); sb.append( ", mergedIndexPath='" ).append( mergedIndexPath ).append( '\'' ); - sb.append( ", mergedIndexTtl='" ).append( mergedIndexTtl ).append( '\'' ); + sb.append( ", mergedIndexTtl=" ).append( mergedIndexTtl ); + sb.append( ", mergedIndexDirectory='" ).append( mergedIndexDirectory ).append( '\'' ); sb.append( '}' ); return sb.toString(); } diff --git a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/RepositoryGroup.java b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/RepositoryGroup.java index 047724dbe..535d309d4 100644 --- a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/RepositoryGroup.java +++ b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/RepositoryGroup.java @@ -51,6 +51,12 @@ public class RepositoryGroup */ private int mergedIndexTtl = 30; + /** + * default model value is empty so none + * @since 2.0.0 + */ + private String mergedIndexCronExpression; + public RepositoryGroup() { // no op @@ -160,6 +166,22 @@ public class RepositoryGroup return this; } + public String getMergedIndexCronExpression() + { + return mergedIndexCronExpression; + } + + public void setMergedIndexCronExpression( String mergedIndexCronExpression ) + { + this.mergedIndexCronExpression = mergedIndexCronExpression; + } + + public RepositoryGroup mergedIndexCronExpression( String mergedIndexCronExpression ) + { + this.mergedIndexCronExpression = mergedIndexCronExpression; + return this; + } + public boolean equals( Object other ) { if ( this == other ) @@ -188,10 +210,12 @@ public class RepositoryGroup @Override public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append( "RepositoryGroup" ); - sb.append( "{id='" ).append( id ).append( '\'' ); + final StringBuilder sb = new StringBuilder( "RepositoryGroup{" ); + sb.append( "id='" ).append( id ).append( '\'' ); sb.append( ", repositories=" ).append( repositories ); + sb.append( ", mergedIndexPath='" ).append( mergedIndexPath ).append( '\'' ); + sb.append( ", mergedIndexTtl=" ).append( mergedIndexTtl ); + sb.append( ", mergedIndexCronExpression='" ).append( mergedIndexCronExpression ).append( '\'' ); sb.append( '}' ); return sb.toString(); } diff --git a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/group/RepositoryGroupAdmin.java b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/group/RepositoryGroupAdmin.java index 5f191d279..007248f6f 100644 --- a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/group/RepositoryGroupAdmin.java +++ b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/group/RepositoryGroupAdmin.java @@ -22,6 +22,7 @@ import org.apache.archiva.admin.model.AuditInformation; import org.apache.archiva.admin.model.RepositoryAdminException; import org.apache.archiva.admin.model.beans.RepositoryGroup; +import java.io.File; import java.util.List; import java.util.Map; @@ -73,4 +74,6 @@ public interface RepositoryGroupAdmin */ Map> getRepositoryToGroupMap() throws RepositoryAdminException; + + File getMergedIndexDirectory( String repositoryGroupId ); } diff --git a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java index ef43c1042..f1e7edd62 100644 --- a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java +++ b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java @@ -33,7 +33,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; import javax.inject.Inject; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -58,6 +60,25 @@ public class DefaultRepositoryGroupAdmin @Inject private ManagedRepositoryAdmin managedRepositoryAdmin; + private File groupsDirectory; + + @PostConstruct + public void initialize() + { + String appServerBase = getRegistry().getString( "appserver.base" ); + groupsDirectory = new File( appServerBase + File.separatorChar + "groups" ); + if ( !groupsDirectory.exists() ) + { + groupsDirectory.mkdirs(); + } + } + + @Override + public File getMergedIndexDirectory( String repositoryGroupId ) + { + return new File( groupsDirectory, repositoryGroupId ); + } + public List getRepositoriesGroups() throws RepositoryAdminException { @@ -68,7 +89,8 @@ public class DefaultRepositoryGroupAdmin { repositoriesGroups.add( new RepositoryGroup( repositoryGroupConfiguration.getId(), new ArrayList( repositoryGroupConfiguration.getRepositories() ) ).mergedIndexPath( - repositoryGroupConfiguration.getMergedIndexPath() ).mergedIndexTtl( repositoryGroupConfiguration.getMergedIndexTtl() ) ); + repositoryGroupConfiguration.getMergedIndexPath() ).mergedIndexTtl( + repositoryGroupConfiguration.getMergedIndexTtl() ) ); } return repositoriesGroups; @@ -285,7 +307,7 @@ public class DefaultRepositoryGroupAdmin "Invalid character(s) found in identifier. Only the following characters are allowed: alphanumeric, '.', '-' and '_'" ); } - if ( repositoryGroup.getMergedIndexTtl() <= 0) + if ( repositoryGroup.getMergedIndexTtl() <= 0 ) { throw new RepositoryAdminException( "Merged Index TTL must be greater than 0." ); } diff --git a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java index 62b94ee25..b6af5e3f9 100644 --- a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java +++ b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java @@ -19,6 +19,7 @@ package org.apache.archiva.webdav; * under the License. */ +import com.google.common.io.Files; import org.apache.archiva.admin.model.RepositoryAdminException; import org.apache.archiva.admin.model.beans.ManagedRepository; import org.apache.archiva.admin.model.beans.RemoteRepository; @@ -37,6 +38,7 @@ import org.apache.archiva.configuration.RepositoryGroupConfiguration; import org.apache.archiva.indexer.merger.IndexMerger; import org.apache.archiva.indexer.merger.IndexMergerException; import org.apache.archiva.indexer.merger.IndexMergerRequest; +import org.apache.archiva.indexer.merger.MergedRemoteIndexesTaskJob; import org.apache.archiva.indexer.merger.TemporaryGroupIndex; import org.apache.archiva.indexer.search.RepositorySearch; import org.apache.archiva.maven2.metadata.MavenMetadataReader; @@ -115,7 +117,7 @@ import java.util.Set; /** * */ -@Service( "davResourceFactory#archiva" ) +@Service("davResourceFactory#archiva") public class ArchivaDavResourceFactory implements DavResourceFactory, Auditable { @@ -134,7 +136,7 @@ public class ArchivaDavResourceFactory private RepositoryRequest repositoryRequest; @Inject - @Named( value = "repositoryProxyConnectors#default" ) + @Named(value = "repositoryProxyConnectors#default") private RepositoryProxyConnectors connectors; @Inject @@ -149,7 +151,7 @@ public class ArchivaDavResourceFactory private ServletAuthenticator servletAuth; @Inject - @Named( value = "httpAuthenticator#basic" ) + @Named(value = "httpAuthenticator#basic") private HttpAuthenticator httpAuth; @Inject @@ -176,11 +178,11 @@ public class ArchivaDavResourceFactory private Digester digestMd5; @Inject - @Named( value = "archivaTaskScheduler#repository" ) + @Named(value = "archivaTaskScheduler#repository") private RepositoryArchivaTaskScheduler scheduler; @Inject - @Named(value= "fileLockManager#default") + @Named(value = "fileLockManager#default") private FileLockManager fileLockManager; private ApplicationContext applicationContext; @@ -345,12 +347,13 @@ public class ArchivaDavResourceFactory if ( metadataChecksum.exists() ) { LogicalResource logicalResource = - new LogicalResource (getLogicalResource( archivaLocator, null, false ) ); + new LogicalResource( getLogicalResource( archivaLocator, null, false ) ); resource = new ArchivaDavResource( metadataChecksum.getAbsolutePath(), logicalResource.getPath(), null, request.getRemoteAddr(), activePrincipal, request.getDavSession(), - archivaLocator, this, mimeTypes, auditListeners, scheduler, fileLockManager ); + archivaLocator, this, mimeTypes, auditListeners, scheduler, + fileLockManager ); } } else @@ -383,7 +386,8 @@ public class ArchivaDavResourceFactory { File resourceFile = writeMergedMetadataToFile( mergedMetadata, filePath ); - LogicalResource logicalResource = new LogicalResource( getLogicalResource( archivaLocator, null, false ) ); + LogicalResource logicalResource = + new LogicalResource( getLogicalResource( archivaLocator, null, false ) ); resource = new ArchivaDavResource( resourceFile.getAbsolutePath(), logicalResource.getPath(), null, @@ -404,7 +408,8 @@ public class ArchivaDavResourceFactory catch ( DigesterException de ) { throw new DavException( HttpServletResponse.SC_INTERNAL_SERVER_ERROR, - "Error occurred while generating checksum files." + de.getMessage() ); + "Error occurred while generating checksum files." + + de.getMessage() ); } } } @@ -445,7 +450,8 @@ public class ArchivaDavResourceFactory File resourceFile = new File( temporaryIndexDirectory, requestedFileName ); resource = new ArchivaDavResource( resourceFile.getAbsolutePath(), requestedFileName, null, request.getRemoteAddr(), activePrincipal, request.getDavSession(), - archivaLocator, this, mimeTypes, auditListeners, scheduler, fileLockManager ); + archivaLocator, this, mimeTypes, auditListeners, scheduler, + fileLockManager ); } else @@ -523,32 +529,39 @@ public class ArchivaDavResourceFactory { // FIXME remove this hack // but currently managedRepository can be null in case of group - String layout = managedRepository == null ? new ManagedRepository( ).getLayout() : managedRepository.getLayout(); + String layout = managedRepository == null ? new ManagedRepository().getLayout() : managedRepository.getLayout(); RepositoryStorage repositoryStorage = this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class ); String path = repositoryStorage.getFilePath( - useOrigResourcePath ? archivaLocator.getOrigResourcePath() : archivaLocator.getResourcePath(), managedRepository ); + useOrigResourcePath ? archivaLocator.getOrigResourcePath() : archivaLocator.getResourcePath(), + managedRepository ); log.debug( "found path {} for resourcePath: '{}' with managedRepo '{}' and layout '{}'", path, - archivaLocator.getResourcePath(), managedRepository == null ? "null" : managedRepository.getId(), layout ); + archivaLocator.getResourcePath(), managedRepository == null ? "null" : managedRepository.getId(), + layout ); return path; } - private String evaluatePathWithVersion( ArchivaDavResourceLocator archivaLocator, ManagedRepositoryContent managedRepositoryContent, String contextPath ) + private String evaluatePathWithVersion( ArchivaDavResourceLocator archivaLocator, + ManagedRepositoryContent managedRepositoryContent, String contextPath ) throws DavException { - String layout = managedRepositoryContent.getRepository() == null ? new ManagedRepository( ).getLayout() : managedRepositoryContent.getRepository().getLayout(); + String layout = managedRepositoryContent.getRepository() == null + ? new ManagedRepository().getLayout() + : managedRepositoryContent.getRepository().getLayout(); RepositoryStorage repositoryStorage = this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class ); try { - return repositoryStorage.getFilePathWithVersion( archivaLocator.getResourcePath(), managedRepositoryContent ); + return repositoryStorage.getFilePathWithVersion( archivaLocator.getResourcePath(), + managedRepositoryContent ); } catch ( RelocationException e ) { String path = e.getPath(); log.debug( "Relocation to {}", path ); - throw new BrowserRedirectException(contextPath + ( StringUtils.startsWith( path, "/" ) ? "": "/" ) + path, e.getRelocationType() ); + throw new BrowserRedirectException( contextPath + ( StringUtils.startsWith( path, "/" ) ? "" : "/" ) + path, + e.getRelocationType() ); } catch ( XMLException e ) { @@ -566,7 +579,7 @@ public class ArchivaDavResourceFactory if ( isAuthorized( request, managedRepositoryContent.getId() ) ) { // Maven Centric part ask evaluation if -SNAPSHOT - String path = evaluatePathWithVersion(archivaLocator, managedRepositoryContent, request.getContextPath()); + String path = evaluatePathWithVersion( archivaLocator, managedRepositoryContent, request.getContextPath() ); if ( path.startsWith( "/" ) ) { path = path.substring( 1 ); @@ -1272,12 +1285,14 @@ public class ArchivaDavResourceFactory } Set authzRepos = new HashSet(); + + String permission = WebdavMethodUtil.getMethodPermission( request.getMethod() ); + for ( String repository : repositories ) { try { - if ( servletAuth.isAuthorized( activePrincipal, repository, - WebdavMethodUtil.getMethodPermission( request.getMethod() ) ) ) + if ( servletAuth.isAuthorized( activePrincipal, repository, permission ) ) { authzRepos.add( repository ); authzRepos.addAll( this.repositorySearch.getRemoteIndexingContextIds( repository ) ); @@ -1293,10 +1308,22 @@ public class ArchivaDavResourceFactory } log.info( "generate temporary merged index for repository group '{}' for repositories '{}'", repositoryGroupConfiguration.getId(), authzRepos ); - IndexingContext indexingContext = indexMerger.buildMergedIndex( - new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(), - repositoryGroupConfiguration.getMergedIndexPath(), - repositoryGroupConfiguration.getMergedIndexTtl() ) ); + + File tempRepoFile = Files.createTempDir(); + tempRepoFile.deleteOnExit(); + + IndexMergerRequest indexMergerRequest = new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(), + repositoryGroupConfiguration.getMergedIndexPath(), + repositoryGroupConfiguration.getMergedIndexTtl() ).mergedIndexDirectory( + tempRepoFile ); + + MergedRemoteIndexesTaskJob job = new MergedRemoteIndexesTaskJob(); + + MergedRemoteIndexesTaskJob.MergedRemoteIndexesTaskRequest taskRequest = + new MergedRemoteIndexesTaskJob.MergedRemoteIndexesTaskRequest(indexMergerRequest, indexMerger); + + IndexingContext indexingContext = job.execute( taskRequest ).getIndexingContext(); + File mergedRepoDir = indexingContext.getIndexDirectoryFile(); TemporaryGroupIndex temporaryGroupIndex = new TemporaryGroupIndex( mergedRepoDir, indexingContext.getId(), repositoryGroupConfiguration.getId(),