[MRM-1746] build merged index for groups with a cron schedule

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1551473 13f79535-47bb-0310-9956-ffa450edef68

archiva-modules/archiva-base/archiva-indexer/pom.xml

@@ -198,6 +198,7 @@
               org.apache.maven.index*,
               org.apache.archiva.redback.components.scheduler,
               org.quartz,
+              org.apache.archiva.scheduler,
               org.slf4j;resolution:=optional
             </Import-Package>
           </instructions>

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultMergedRemoteIndexesScheduler.java

@@ -0,0 +1,96 @@
+package org.apache.archiva.indexer.merger;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.archiva.admin.model.beans.RepositoryGroup;
+import org.apache.archiva.admin.model.group.RepositoryGroupAdmin;
+import org.apache.archiva.scheduler.MergedRemoteIndexesScheduler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.support.CronTrigger;
+import org.springframework.stereotype.Service;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledFuture;
+
+/**
+ * @author Olivier Lamy
+ * @since 2.0.0
+ */
+@Service( "mergedRemoteIndexesScheduler#default" )
+public class DefaultMergedRemoteIndexesScheduler
+    implements MergedRemoteIndexesScheduler
+{
+
+    private Logger logger = LoggerFactory.getLogger( getClass() );
+
+    @Inject
+    @Named( value = "taskScheduler#mergeRemoteIndexes" )
+    private TaskScheduler taskScheduler;
+
+    @Inject
+    private RepositoryGroupAdmin repositoryGroupAdmin;
+
+    @Inject
+    private IndexMerger indexMerger;
+
+    private Map<String, ScheduledFuture> scheduledFutureMap = new ConcurrentHashMap<String, ScheduledFuture>();
+
+    @Override
+    public void schedule( RepositoryGroup repositoryGroup )
+    {
+        if ( repositoryGroup.getCronExpression() == null )
+        {
+            return;
+        }
+        CronTrigger cronTrigger = new CronTrigger( repositoryGroup.getCronExpression() );
+
+        File directory = repositoryGroupAdmin.getMergedIndexDirectory( repositoryGroup.getId() );
+
+        List<String> repositories = repositoryGroup.getRepositories();
+
+        IndexMergerRequest indexMergerRequest =
+            new IndexMergerRequest( repositories, true, repositoryGroup.getId(), repositoryGroup.getMergedIndexPath(),
+                                    repositoryGroup.getMergedIndexTtl() ).mergedIndexDirectory( directory );
+
+        MergedRemoteIndexesTask.MergedRemoteIndexesTaskRequest taskRequest =
+            new MergedRemoteIndexesTask.MergedRemoteIndexesTaskRequest( indexMergerRequest, indexMerger );
+
+        ScheduledFuture scheduledFuture =
+            taskScheduler.schedule( new MergedRemoteIndexesTask( taskRequest ), cronTrigger );
+        scheduledFutureMap.put( repositoryGroup.getId(), scheduledFuture );
+    }
+
+    @Override
+    public void unschedule( RepositoryGroup repositoryGroup )
+    {
+        ScheduledFuture scheduledFuture = scheduledFutureMap.remove( repositoryGroup.getId() );
+        if ( scheduledFuture != null )
+        {
+            scheduledFuture.cancel( true );
+        }
+    }
+}

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java

@@ -147,4 +147,32 @@ public class IndexMergerRequest
         sb.append( '}' );
         return sb.toString();
     }
+
+    @Override
+    public boolean equals( Object o )
+    {
+        if ( this == o )
+        {
+            return true;
+        }
+        if ( o == null || getClass() != o.getClass() )
+        {
+            return false;
+        }
+
+        IndexMergerRequest that = (IndexMergerRequest) o;
+
+        if ( !groupId.equals( that.groupId ) )
+        {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return groupId.hashCode();
+    }
 }

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/MergedRemoteIndexesTask.java

@@ -19,15 +19,10 @@ package org.apache.archiva.indexer.merger;
  * under the License.
  */
 
-import org.apache.archiva.redback.components.scheduler.AbstractJob;
 import org.apache.maven.index.context.IndexingContext;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-
 /**
  * @author Olivier Lamy
  * @since 2.0.0
@@ -58,7 +53,7 @@ public class MergedRemoteIndexesTask
         }
     }
 
-    public MergedRemoteIndexesTaskResult execute( )
+    public MergedRemoteIndexesTaskResult execute()
         throws IndexMergerException
     {
         IndexMerger indexMerger = mergedRemoteIndexesTaskRequest.indexMerger;
@@ -100,6 +95,34 @@ public class MergedRemoteIndexesTask
         {
             this.indexMerger = indexMerger;
         }
+
+        @Override
+        public boolean equals( Object o )
+        {
+            if ( this == o )
+            {
+                return true;
+            }
+            if ( o == null || getClass() != o.getClass() )
+            {
+                return false;
+            }
+
+            MergedRemoteIndexesTaskRequest that = (MergedRemoteIndexesTaskRequest) o;
+
+            if ( !indexMergerRequest.equals( that.indexMergerRequest ) )
+            {
+                return false;
+            }
+
+            return true;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return indexMergerRequest.hashCode();
+        }
     }
 
     public static class MergedRemoteIndexesTaskResult
@@ -121,4 +144,32 @@ public class MergedRemoteIndexesTask
             this.indexingContext = indexingContext;
         }
     }
+
+    @Override
+    public boolean equals( Object o )
+    {
+        if ( this == o )
+        {
+            return true;
+        }
+        if ( !( o instanceof MergedRemoteIndexesTask ) )
+        {
+            return false;
+        }
+
+        MergedRemoteIndexesTask that = (MergedRemoteIndexesTask) o;
+
+        if ( !mergedRemoteIndexesTaskRequest.equals( that.mergedRemoteIndexesTaskRequest ) )
+        {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return mergedRemoteIndexesTaskRequest.hashCode();
+    }
 }

archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml

@@ -40,5 +40,10 @@
     </constructor-arg>
   </bean>
 
+  <bean name="taskScheduler#mergeRemoteIndexes"
+        class="org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler">
+    <property name="poolSize" value="4"/>
+    <property name="threadGroupName" value="mergeRemoteIndexes"/>
+  </bean>
 
 </beans>

archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java

@@ -27,7 +27,11 @@ import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
 import org.apache.archiva.admin.repository.AbstractRepositoryAdmin;
 import org.apache.archiva.audit.AuditEvent;
 import org.apache.archiva.configuration.Configuration;
+import org.apache.archiva.configuration.ConfigurationEvent;
+import org.apache.archiva.configuration.ConfigurationListener;
 import org.apache.archiva.configuration.RepositoryGroupConfiguration;
+import org.apache.archiva.redback.components.registry.RegistryListener;
+import org.apache.archiva.scheduler.MergedRemoteIndexesScheduler;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,6 +42,7 @@ import javax.inject.Inject;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -60,6 +65,9 @@ public class DefaultRepositoryGroupAdmin
     @Inject
     private ManagedRepositoryAdmin managedRepositoryAdmin;
 
+    @Inject
+    private MergedRemoteIndexesScheduler mergedRemoteIndexesScheduler;
+
     private File groupsDirectory;
 
     @PostConstruct
@@ -71,8 +79,22 @@ public class DefaultRepositoryGroupAdmin
         {
             groupsDirectory.mkdirs();
         }
+
+        try
+        {
+            for ( RepositoryGroup repositoryGroup : getRepositoriesGroups() )
+            {
+                mergedRemoteIndexesScheduler.schedule( repositoryGroup );
+            }
+        }
+        catch ( RepositoryAdminException e )
+        {
+            log.warn( "fail to getRepositoriesGroups {}", e.getMessage(), e );
+        }
+
     }
 
+
     @Override
     public File getMergedIndexDirectory( String repositoryGroupId )
     {
@@ -121,10 +143,12 @@ public class DefaultRepositoryGroupAdmin
         repositoryGroupConfiguration.setRepositories( repositoryGroup.getRepositories() );
         repositoryGroupConfiguration.setMergedIndexPath( repositoryGroup.getMergedIndexPath() );
         repositoryGroupConfiguration.setMergedIndexTtl( repositoryGroup.getMergedIndexTtl() );
+        repositoryGroupConfiguration.setCronExpression( repositoryGroup.getCronExpression() );
         Configuration configuration = getArchivaConfiguration().getConfiguration();
         configuration.addRepositoryGroup( repositoryGroupConfiguration );
         saveConfiguration( configuration );
         triggerAuditEvent( repositoryGroup.getId(), null, AuditEvent.ADD_REPO_GROUP, auditInformation );
+        mergedRemoteIndexesScheduler.schedule( repositoryGroup );
         return Boolean.TRUE;
     }
 
@@ -134,6 +158,8 @@ public class DefaultRepositoryGroupAdmin
         Configuration configuration = getArchivaConfiguration().getConfiguration();
         RepositoryGroupConfiguration repositoryGroupConfiguration =
             configuration.getRepositoryGroupsAsMap().get( repositoryGroupId );
+        mergedRemoteIndexesScheduler.unschedule(
+            new RepositoryGroup( repositoryGroupId, Collections.<String>emptyList() ) );
         if ( repositoryGroupConfiguration == null )
         {
             throw new RepositoryAdminException(
@@ -141,6 +167,7 @@ public class DefaultRepositoryGroupAdmin
         }
         configuration.removeRepositoryGroup( repositoryGroupConfiguration );
         triggerAuditEvent( repositoryGroupId, null, AuditEvent.DELETE_REPO_GROUP, auditInformation );
+
         return Boolean.TRUE;
     }
 
@@ -166,6 +193,7 @@ public class DefaultRepositoryGroupAdmin
         repositoryGroupConfiguration.setRepositories( repositoryGroup.getRepositories() );
         repositoryGroupConfiguration.setMergedIndexPath( repositoryGroup.getMergedIndexPath() );
         repositoryGroupConfiguration.setMergedIndexTtl( repositoryGroup.getMergedIndexTtl() );
+        repositoryGroupConfiguration.setCronExpression( repositoryGroup.getCronExpression() );
         configuration.addRepositoryGroup( repositoryGroupConfiguration );
 
         saveConfiguration( configuration );
@@ -173,6 +201,8 @@ public class DefaultRepositoryGroupAdmin
         {
             triggerAuditEvent( repositoryGroup.getId(), null, AuditEvent.MODIFY_REPO_GROUP, auditInformation );
         }
+        mergedRemoteIndexesScheduler.unschedule( repositoryGroup );
+        mergedRemoteIndexesScheduler.schedule( repositoryGroup );
         return Boolean.TRUE;
     }
 

archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/test/java/org/apache/archiva/admin/mock/MockMergedRemoteIndexesScheduler.java

@@ -0,0 +1,45 @@
+package org.apache.archiva.admin.mock;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.archiva.admin.model.beans.RepositoryGroup;
+import org.apache.archiva.scheduler.MergedRemoteIndexesScheduler;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author Olivier Lamy
+ */
+@Service
+public class MockMergedRemoteIndexesScheduler
+    implements MergedRemoteIndexesScheduler
+{
+
+    @Override
+    public void schedule( RepositoryGroup repositoryGroup )
+    {
+        // no op
+    }
+
+    @Override
+    public void unschedule( RepositoryGroup repositoryGroup )
+    {
+        // no op
+    }
+}

archiva-modules/archiva-scheduler/archiva-scheduler-api/pom.xml

@@ -51,6 +51,7 @@
             </Export-Package>
             <Import-Package>
               org.apache.archiva.redback.components.taskqueue,
+              org.apache.archiva.admin.model.beans
             </Import-Package>
           </instructions>
         </configuration>

archiva-modules/archiva-scheduler/archiva-scheduler-api/src/main/java/org/apache/archiva/scheduler/MergedRemoteIndexesScheduler.java

@@ -0,0 +1,40 @@
+package org.apache.archiva.scheduler;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.archiva.admin.model.beans.RepositoryGroup;
+
+/**
+ * @author Olivier Lamy
+ * @since 2.0.0
+ */
+public interface MergedRemoteIndexesScheduler
+{
+
+    /**
+     * will check if this repository group need to a schedule a cron to download/merge
+     * remote indexes
+     * @param repositoryGroup
+     */
+    void schedule( RepositoryGroup repositoryGroup );
+
+    void unschedule( RepositoryGroup repositoryGroup );
+
+}

archiva-modules/archiva-scheduler/archiva-scheduler-repository-api/src/main/java/org/apache/archiva/scheduler/repository/model/RepositoryArchivaTaskScheduler.java

@@ -29,10 +29,8 @@ public interface RepositoryArchivaTaskScheduler
     extends ArchivaTaskScheduler<RepositoryTask>
 {
 
-
     boolean isProcessingRepositoryTask( String repositoryId );
 
-
     boolean isProcessingRepositoryTask( RepositoryTask task );
 
     void queueTask( RepositoryTask task )

archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java

@@ -1313,10 +1313,7 @@ public class ArchivaDavResourceFactory
 
             IndexMergerRequest indexMergerRequest = new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(),
                                     repositoryGroupConfiguration.getMergedIndexPath(),
-                                    repositoryGroupConfiguration.getMergedIndexTtl() ).mergedIndexDirectory(
-                tempRepoFile );
-
-
+                                    repositoryGroupConfiguration.getMergedIndexTtl() ).mergedIndexDirectory( tempRepoFile );
 
             MergedRemoteIndexesTask.MergedRemoteIndexesTaskRequest taskRequest =
                 new MergedRemoteIndexesTask.MergedRemoteIndexesTaskRequest(indexMergerRequest, indexMerger);