From af520c00cfe4ffb0790a30e916f538b84ee8667b Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Tue, 17 Dec 2013 07:28:54 +0000 Subject: [PATCH] [MRM-1746] build merged index for groups with a cron schedule git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1551473 13f79535-47bb-0310-9956-ffa450edef68 --- .../archiva-base/archiva-indexer/pom.xml | 1 + .../DefaultMergedRemoteIndexesScheduler.java | 96 +++++++++++++++++++ .../indexer/merger/IndexMergerRequest.java | 28 ++++++ .../merger/MergedRemoteIndexesTask.java | 63 ++++++++++-- .../resources/META-INF/spring-context.xml | 5 + .../group/DefaultRepositoryGroupAdmin.java | 30 ++++++ .../MockMergedRemoteIndexesScheduler.java | 45 +++++++++ .../archiva-scheduler-api/pom.xml | 1 + .../MergedRemoteIndexesScheduler.java | 40 ++++++++ .../model/RepositoryArchivaTaskScheduler.java | 2 - .../webdav/ArchivaDavResourceFactory.java | 5 +- 11 files changed, 304 insertions(+), 12 deletions(-) create mode 100644 archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultMergedRemoteIndexesScheduler.java create mode 100644 archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/test/java/org/apache/archiva/admin/mock/MockMergedRemoteIndexesScheduler.java create mode 100644 archiva-modules/archiva-scheduler/archiva-scheduler-api/src/main/java/org/apache/archiva/scheduler/MergedRemoteIndexesScheduler.java diff --git a/archiva-modules/archiva-base/archiva-indexer/pom.xml b/archiva-modules/archiva-base/archiva-indexer/pom.xml index 6579897fe..6eea49d2f 100644 --- a/archiva-modules/archiva-base/archiva-indexer/pom.xml +++ b/archiva-modules/archiva-base/archiva-indexer/pom.xml @@ -198,6 +198,7 @@ org.apache.maven.index*, org.apache.archiva.redback.components.scheduler, org.quartz, + org.apache.archiva.scheduler, org.slf4j;resolution:=optional diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultMergedRemoteIndexesScheduler.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultMergedRemoteIndexesScheduler.java new file mode 100644 index 000000000..787c1c69e --- /dev/null +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultMergedRemoteIndexesScheduler.java @@ -0,0 +1,96 @@ +package org.apache.archiva.indexer.merger; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.admin.model.beans.RepositoryGroup; +import org.apache.archiva.admin.model.group.RepositoryGroupAdmin; +import org.apache.archiva.scheduler.MergedRemoteIndexesScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.support.CronTrigger; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.inject.Named; +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledFuture; + +/** + * @author Olivier Lamy + * @since 2.0.0 + */ +@Service( "mergedRemoteIndexesScheduler#default" ) +public class DefaultMergedRemoteIndexesScheduler + implements MergedRemoteIndexesScheduler +{ + + private Logger logger = LoggerFactory.getLogger( getClass() ); + + @Inject + @Named( value = "taskScheduler#mergeRemoteIndexes" ) + private TaskScheduler taskScheduler; + + @Inject + private RepositoryGroupAdmin repositoryGroupAdmin; + + @Inject + private IndexMerger indexMerger; + + private Map scheduledFutureMap = new ConcurrentHashMap(); + + @Override + public void schedule( RepositoryGroup repositoryGroup ) + { + if ( repositoryGroup.getCronExpression() == null ) + { + return; + } + CronTrigger cronTrigger = new CronTrigger( repositoryGroup.getCronExpression() ); + + File directory = repositoryGroupAdmin.getMergedIndexDirectory( repositoryGroup.getId() ); + + List repositories = repositoryGroup.getRepositories(); + + IndexMergerRequest indexMergerRequest = + new IndexMergerRequest( repositories, true, repositoryGroup.getId(), repositoryGroup.getMergedIndexPath(), + repositoryGroup.getMergedIndexTtl() ).mergedIndexDirectory( directory ); + + MergedRemoteIndexesTask.MergedRemoteIndexesTaskRequest taskRequest = + new MergedRemoteIndexesTask.MergedRemoteIndexesTaskRequest( indexMergerRequest, indexMerger ); + + ScheduledFuture scheduledFuture = + taskScheduler.schedule( new MergedRemoteIndexesTask( taskRequest ), cronTrigger ); + scheduledFutureMap.put( repositoryGroup.getId(), scheduledFuture ); + } + + @Override + public void unschedule( RepositoryGroup repositoryGroup ) + { + ScheduledFuture scheduledFuture = scheduledFutureMap.remove( repositoryGroup.getId() ); + if ( scheduledFuture != null ) + { + scheduledFuture.cancel( true ); + } + } +} diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java index e99e3d562..65ceb17d0 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java @@ -147,4 +147,32 @@ public class IndexMergerRequest sb.append( '}' ); return sb.toString(); } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( o == null || getClass() != o.getClass() ) + { + return false; + } + + IndexMergerRequest that = (IndexMergerRequest) o; + + if ( !groupId.equals( that.groupId ) ) + { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + return groupId.hashCode(); + } } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/MergedRemoteIndexesTask.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/MergedRemoteIndexesTask.java index 2e19eb640..346d08f4b 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/MergedRemoteIndexesTask.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/MergedRemoteIndexesTask.java @@ -19,15 +19,10 @@ package org.apache.archiva.indexer.merger; * under the License. */ -import org.apache.archiva.redback.components.scheduler.AbstractJob; import org.apache.maven.index.context.IndexingContext; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; - /** * @author Olivier Lamy * @since 2.0.0 @@ -58,7 +53,7 @@ public class MergedRemoteIndexesTask } } - public MergedRemoteIndexesTaskResult execute( ) + public MergedRemoteIndexesTaskResult execute() throws IndexMergerException { IndexMerger indexMerger = mergedRemoteIndexesTaskRequest.indexMerger; @@ -100,6 +95,34 @@ public class MergedRemoteIndexesTask { this.indexMerger = indexMerger; } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( o == null || getClass() != o.getClass() ) + { + return false; + } + + MergedRemoteIndexesTaskRequest that = (MergedRemoteIndexesTaskRequest) o; + + if ( !indexMergerRequest.equals( that.indexMergerRequest ) ) + { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + return indexMergerRequest.hashCode(); + } } public static class MergedRemoteIndexesTaskResult @@ -121,4 +144,32 @@ public class MergedRemoteIndexesTask this.indexingContext = indexingContext; } } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( !( o instanceof MergedRemoteIndexesTask ) ) + { + return false; + } + + MergedRemoteIndexesTask that = (MergedRemoteIndexesTask) o; + + if ( !mergedRemoteIndexesTaskRequest.equals( that.mergedRemoteIndexesTaskRequest ) ) + { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + return mergedRemoteIndexesTaskRequest.hashCode(); + } } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml b/archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml index 4beb0621c..8c0de11db 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml @@ -40,5 +40,10 @@ + + + + \ No newline at end of file diff --git a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java index f1e7edd62..e35c74c06 100644 --- a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java +++ b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java @@ -27,7 +27,11 @@ import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; import org.apache.archiva.admin.repository.AbstractRepositoryAdmin; import org.apache.archiva.audit.AuditEvent; import org.apache.archiva.configuration.Configuration; +import org.apache.archiva.configuration.ConfigurationEvent; +import org.apache.archiva.configuration.ConfigurationListener; import org.apache.archiva.configuration.RepositoryGroupConfiguration; +import org.apache.archiva.redback.components.registry.RegistryListener; +import org.apache.archiva.scheduler.MergedRemoteIndexesScheduler; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +42,7 @@ import javax.inject.Inject; import java.io.File; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -60,6 +65,9 @@ public class DefaultRepositoryGroupAdmin @Inject private ManagedRepositoryAdmin managedRepositoryAdmin; + @Inject + private MergedRemoteIndexesScheduler mergedRemoteIndexesScheduler; + private File groupsDirectory; @PostConstruct @@ -71,8 +79,22 @@ public class DefaultRepositoryGroupAdmin { groupsDirectory.mkdirs(); } + + try + { + for ( RepositoryGroup repositoryGroup : getRepositoriesGroups() ) + { + mergedRemoteIndexesScheduler.schedule( repositoryGroup ); + } + } + catch ( RepositoryAdminException e ) + { + log.warn( "fail to getRepositoriesGroups {}", e.getMessage(), e ); + } + } + @Override public File getMergedIndexDirectory( String repositoryGroupId ) { @@ -121,10 +143,12 @@ public class DefaultRepositoryGroupAdmin repositoryGroupConfiguration.setRepositories( repositoryGroup.getRepositories() ); repositoryGroupConfiguration.setMergedIndexPath( repositoryGroup.getMergedIndexPath() ); repositoryGroupConfiguration.setMergedIndexTtl( repositoryGroup.getMergedIndexTtl() ); + repositoryGroupConfiguration.setCronExpression( repositoryGroup.getCronExpression() ); Configuration configuration = getArchivaConfiguration().getConfiguration(); configuration.addRepositoryGroup( repositoryGroupConfiguration ); saveConfiguration( configuration ); triggerAuditEvent( repositoryGroup.getId(), null, AuditEvent.ADD_REPO_GROUP, auditInformation ); + mergedRemoteIndexesScheduler.schedule( repositoryGroup ); return Boolean.TRUE; } @@ -134,6 +158,8 @@ public class DefaultRepositoryGroupAdmin Configuration configuration = getArchivaConfiguration().getConfiguration(); RepositoryGroupConfiguration repositoryGroupConfiguration = configuration.getRepositoryGroupsAsMap().get( repositoryGroupId ); + mergedRemoteIndexesScheduler.unschedule( + new RepositoryGroup( repositoryGroupId, Collections.emptyList() ) ); if ( repositoryGroupConfiguration == null ) { throw new RepositoryAdminException( @@ -141,6 +167,7 @@ public class DefaultRepositoryGroupAdmin } configuration.removeRepositoryGroup( repositoryGroupConfiguration ); triggerAuditEvent( repositoryGroupId, null, AuditEvent.DELETE_REPO_GROUP, auditInformation ); + return Boolean.TRUE; } @@ -166,6 +193,7 @@ public class DefaultRepositoryGroupAdmin repositoryGroupConfiguration.setRepositories( repositoryGroup.getRepositories() ); repositoryGroupConfiguration.setMergedIndexPath( repositoryGroup.getMergedIndexPath() ); repositoryGroupConfiguration.setMergedIndexTtl( repositoryGroup.getMergedIndexTtl() ); + repositoryGroupConfiguration.setCronExpression( repositoryGroup.getCronExpression() ); configuration.addRepositoryGroup( repositoryGroupConfiguration ); saveConfiguration( configuration ); @@ -173,6 +201,8 @@ public class DefaultRepositoryGroupAdmin { triggerAuditEvent( repositoryGroup.getId(), null, AuditEvent.MODIFY_REPO_GROUP, auditInformation ); } + mergedRemoteIndexesScheduler.unschedule( repositoryGroup ); + mergedRemoteIndexesScheduler.schedule( repositoryGroup ); return Boolean.TRUE; } diff --git a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/test/java/org/apache/archiva/admin/mock/MockMergedRemoteIndexesScheduler.java b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/test/java/org/apache/archiva/admin/mock/MockMergedRemoteIndexesScheduler.java new file mode 100644 index 000000000..54552a53c --- /dev/null +++ b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/test/java/org/apache/archiva/admin/mock/MockMergedRemoteIndexesScheduler.java @@ -0,0 +1,45 @@ +package org.apache.archiva.admin.mock; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.admin.model.beans.RepositoryGroup; +import org.apache.archiva.scheduler.MergedRemoteIndexesScheduler; +import org.springframework.stereotype.Service; + +/** + * @author Olivier Lamy + */ +@Service +public class MockMergedRemoteIndexesScheduler + implements MergedRemoteIndexesScheduler +{ + + @Override + public void schedule( RepositoryGroup repositoryGroup ) + { + // no op + } + + @Override + public void unschedule( RepositoryGroup repositoryGroup ) + { + // no op + } +} diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-api/pom.xml b/archiva-modules/archiva-scheduler/archiva-scheduler-api/pom.xml index 52e6b49d1..18e7d3112 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-api/pom.xml +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-api/pom.xml @@ -51,6 +51,7 @@ org.apache.archiva.redback.components.taskqueue, + org.apache.archiva.admin.model.beans diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-api/src/main/java/org/apache/archiva/scheduler/MergedRemoteIndexesScheduler.java b/archiva-modules/archiva-scheduler/archiva-scheduler-api/src/main/java/org/apache/archiva/scheduler/MergedRemoteIndexesScheduler.java new file mode 100644 index 000000000..ae96cfe64 --- /dev/null +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-api/src/main/java/org/apache/archiva/scheduler/MergedRemoteIndexesScheduler.java @@ -0,0 +1,40 @@ +package org.apache.archiva.scheduler; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.admin.model.beans.RepositoryGroup; + +/** + * @author Olivier Lamy + * @since 2.0.0 + */ +public interface MergedRemoteIndexesScheduler +{ + + /** + * will check if this repository group need to a schedule a cron to download/merge + * remote indexes + * @param repositoryGroup + */ + void schedule( RepositoryGroup repositoryGroup ); + + void unschedule( RepositoryGroup repositoryGroup ); + +} diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-repository-api/src/main/java/org/apache/archiva/scheduler/repository/model/RepositoryArchivaTaskScheduler.java b/archiva-modules/archiva-scheduler/archiva-scheduler-repository-api/src/main/java/org/apache/archiva/scheduler/repository/model/RepositoryArchivaTaskScheduler.java index 157eee15f..56c684a7e 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-repository-api/src/main/java/org/apache/archiva/scheduler/repository/model/RepositoryArchivaTaskScheduler.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-repository-api/src/main/java/org/apache/archiva/scheduler/repository/model/RepositoryArchivaTaskScheduler.java @@ -29,10 +29,8 @@ public interface RepositoryArchivaTaskScheduler extends ArchivaTaskScheduler { - boolean isProcessingRepositoryTask( String repositoryId ); - boolean isProcessingRepositoryTask( RepositoryTask task ); void queueTask( RepositoryTask task ) diff --git a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java index cbe1172ba..181eddc63 100644 --- a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java +++ b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java @@ -1313,10 +1313,7 @@ public class ArchivaDavResourceFactory IndexMergerRequest indexMergerRequest = new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(), repositoryGroupConfiguration.getMergedIndexPath(), - repositoryGroupConfiguration.getMergedIndexTtl() ).mergedIndexDirectory( - tempRepoFile ); - - + repositoryGroupConfiguration.getMergedIndexTtl() ).mergedIndexDirectory( tempRepoFile ); MergedRemoteIndexesTask.MergedRemoteIndexesTaskRequest taskRequest = new MergedRemoteIndexesTask.MergedRemoteIndexesTaskRequest(indexMergerRequest, indexMerger); -- 2.39.5