From 7ad0e303a0b77c9abb8197b4dd528c6f90bd74b3 Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Sat, 5 Nov 2011 18:50:28 +0000 Subject: [MRM-815] aggregate indices for repository groups. delete temporary on session end with a session listener periodical taks to cleanup too old temp group index. git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1198011 13f79535-47bb-0310-9956-ffa450edef68 --- .../archiva/indexer/merger/DefaultIndexMerger.java | 84 +++++-------------- .../apache/archiva/indexer/merger/IndexMerger.java | 15 +++- .../indexer/merger/TemporaryGroupIndex.java | 95 ++++++++++++++++++++++ .../indexer/merger/TemporaryGroupIndexCleaner.java | 85 +++++++++++++++++++ 4 files changed, 214 insertions(+), 65 deletions(-) create mode 100644 archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java create mode 100644 archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java (limited to 'archiva-modules/archiva-base/archiva-indexer') diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java index 9c0613490..674f261fb 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java @@ -23,7 +23,6 @@ import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; import org.apache.archiva.common.plexusbridge.MavenIndexerUtils; import org.apache.archiva.common.plexusbridge.PlexusSisuBridge; import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException; -import org.apache.commons.io.FileUtils; import org.apache.maven.index.NexusIndexer; import org.apache.maven.index.context.IndexingContext; import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; @@ -31,14 +30,13 @@ import org.apache.maven.index.packer.IndexPacker; import org.apache.maven.index.packer.IndexPackingRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import javax.inject.Inject; import java.io.File; import java.io.IOException; import java.util.Collection; -import java.util.Date; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -62,7 +60,7 @@ public class DefaultIndexMerger private IndexPacker indexPacker; - private List temporaryIndexes = new CopyOnWriteArrayList(); + private List temporaryGroupIndexes = new CopyOnWriteArrayList(); @Inject public DefaultIndexMerger( PlexusSisuBridge plexusSisuBridge, MavenIndexerUtils mavenIndexerUtils ) @@ -73,7 +71,7 @@ public class DefaultIndexMerger indexPacker = plexusSisuBridge.lookup( IndexPacker.class, "default" ); } - public File buildMergedIndex( Collection repositoriesIds, boolean packIndex ) + public IndexingContext buildMergedIndex( Collection repositoriesIds, boolean packIndex ) throws IndexMergerException { File tempRepoFile = Files.createTempDir(); @@ -104,8 +102,8 @@ public class DefaultIndexMerger IndexPackingRequest request = new IndexPackingRequest( indexingContext, indexLocation ); indexPacker.packIndex( request ); } - temporaryIndexes.add( new TemporaryIndex( tempRepoFile, tempRepoId ) ); - return indexingContext.getIndexDirectoryFile(); + temporaryGroupIndexes.add( new TemporaryGroupIndex( tempRepoFile, tempRepoId ) ); + return indexingContext; } catch ( IOException e ) { @@ -117,70 +115,30 @@ public class DefaultIndexMerger } } - - @Scheduled( fixedDelay = 900000 ) - public void cleanTemporaryIndex() + @Async + public void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex ) { - for ( TemporaryIndex temporaryIndex : temporaryIndexes ) + if ( temporaryGroupIndex == null ) { - // cleanup files older than 30 minutes - if ( new Date().getTime() - temporaryIndex.creationTime > 1800000 ) - { - try - { - IndexingContext context = indexer.getIndexingContexts().get( temporaryIndex.indexId ); - if ( context != null ) - { - indexer.removeIndexingContext( context, true ); - } - else - { - FileUtils.deleteDirectory( temporaryIndex.directory ); - } - temporaryIndexes.remove( temporaryIndex ); - log.debug( "remove directory {}", temporaryIndex.directory ); - } - catch ( IOException e ) - { - log.warn( "failed to remove directory:" + temporaryIndex.directory, e ); - } - } - temporaryIndexes.remove( temporaryIndex ); + return; } - } - - private static class TemporaryIndex - { - private long creationTime = new Date().getTime(); - private File directory; - - private String indexId; - - TemporaryIndex( File directory, String indexId ) + try { - this.directory = directory; - this.indexId = indexId; + IndexingContext indexingContext = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() ); + if ( indexingContext != null ) + { + indexer.removeIndexingContext( indexingContext, true ); + } } - - @Override - public int hashCode() + catch ( IOException e ) { - return Long.toString( creationTime ).hashCode(); + log.warn( "fail to delete temporary group index {}", temporaryGroupIndex.getIndexId(), e ); } + } - @Override - public boolean equals( Object o ) - { - if ( this == o ) - { - return true; - } - if ( !( o instanceof TemporaryIndex ) ) - { - return false; - } - return this.creationTime == ( (TemporaryIndex) o ).creationTime; - } + public Collection getTemporaryGroupIndexes() + { + return this.temporaryGroupIndexes; } } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java index 89e2c7d6f..17a1f21f6 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java @@ -18,6 +18,8 @@ package org.apache.archiva.indexer.merger; * under the License. */ +import org.apache.maven.index.context.IndexingContext; + import java.io.File; import java.util.Collection; @@ -27,12 +29,21 @@ import java.util.Collection; */ public interface IndexMerger { + /** + * default tmp created group index ttl in minutes + */ + static final int DEFAULT_GROUP_INDEX_TTL = 1; + /** * @param repositoriesIds repositories Ids to merge content - * @param packIndex will generate a downloadable index + * @param packIndex will generate a downloadable index * @return a temporary directory with a merge index (directory marked deleteOnExit) * @throws IndexMergerException */ - File buildMergedIndex( Collection repositoriesIds, boolean packIndex ) + IndexingContext buildMergedIndex( Collection repositoriesIds, boolean packIndex ) throws IndexMergerException; + + void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex ); + + Collection getTemporaryGroupIndexes(); } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java new file mode 100644 index 000000000..489ac1594 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java @@ -0,0 +1,95 @@ +package org.apache.archiva.indexer.merger; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.File; +import java.io.Serializable; +import java.util.Date; + +/** + * @author Olivier Lamy + */ +public class TemporaryGroupIndex + implements Serializable +{ + private long creationTime = new Date().getTime(); + + private File directory; + + private String indexId; + + public TemporaryGroupIndex( File directory, String indexId ) + { + this.directory = directory; + this.indexId = indexId; + } + + public long getCreationTime() + { + return creationTime; + } + + public TemporaryGroupIndex setCreationTime( long creationTime ) + { + this.creationTime = creationTime; + return this; + } + + public File getDirectory() + { + return directory; + } + + public TemporaryGroupIndex setDirectory( File directory ) + { + this.directory = directory; + return this; + } + + public String getIndexId() + { + return indexId; + } + + public TemporaryGroupIndex setIndexId( String indexId ) + { + this.indexId = indexId; + return this; + } + + @Override + public int hashCode() + { + return Long.toString( creationTime ).hashCode(); + } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( !( o instanceof TemporaryGroupIndex ) ) + { + return false; + } + return this.creationTime == ( (TemporaryGroupIndex) o ).creationTime; + } +} diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java new file mode 100644 index 000000000..0845db6b1 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java @@ -0,0 +1,85 @@ +package org.apache.archiva.indexer.merger; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.common.plexusbridge.PlexusSisuBridge; +import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException; +import org.apache.maven.index.NexusIndexer; +import org.apache.maven.index.context.IndexingContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.Date; + +/** + * @author Olivier Lamy + * @since 1.4-M2 + */ +@Service +public class TemporaryGroupIndexCleaner +{ + private Logger log = LoggerFactory.getLogger( getClass() ); + + @Inject + private IndexMerger indexMerger; + + private NexusIndexer indexer; + + @Inject + public TemporaryGroupIndexCleaner( PlexusSisuBridge plexusSisuBridge ) + throws PlexusSisuBridgeException + { + indexer = plexusSisuBridge.lookup( NexusIndexer.class ); + } + + // 900000 + @Scheduled( fixedDelay = 900000 ) + public void cleanTemporaryIndex() + { + for ( TemporaryGroupIndex temporaryGroupIndex : indexMerger.getTemporaryGroupIndexes() ) + { + // cleanup files older than 60 minutes 3600000 + if ( new Date().getTime() - temporaryGroupIndex.getCreationTime() > 3600000 ) + { + try + { + IndexingContext context = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() ); + if ( context != null ) + { + indexer.removeIndexingContext( context, true ); + } + else + { + indexMerger.cleanTemporaryGroupIndex( temporaryGroupIndex ); + } + indexMerger.getTemporaryGroupIndexes().remove( temporaryGroupIndex ); + log.debug( "remove directory {}", temporaryGroupIndex.getDirectory() ); + } + catch ( IOException e ) + { + log.warn( "failed to remove directory:" + temporaryGroupIndex.getDirectory(), e ); + } + } + } + } +} -- cgit v1.2.3