From 1d6f9dd226684b7522b8b2c8d60ffa55c4c7b350 Mon Sep 17 00:00:00 2001 From: Martin Stockhammer Date: Thu, 24 Aug 2017 21:14:04 +0200 Subject: [PATCH] Fixing metadata removal. Adding mocks for verification. - Prevent redundant removals for same artifacts - Adding metadata removal for each file removal - Adding mock verifications for metadata removal in unit tests --- .../repository/AbstractRepositoryPurge.java | 363 ++++++++++++++---- .../repository/ArtifactFilenameFilter.java | 49 --- ...eanupReleasedSnapshotsRepositoryPurge.java | 8 +- .../repository/DaysOldRepositoryPurge.java | 26 +- .../repository/RepositoryPurgeConsumer.java | 4 - .../RetentionCountRepositoryPurge.java | 27 +- .../AbstractRepositoryPurgeTest.java | 25 ++ ...pReleasedSnapshotsRepositoryPurgeTest.java | 93 ++++- .../DaysOldRepositoryPurgeTest.java | 103 ++++- .../RepositoryPurgeConsumerTest.java | 115 +++++- .../RetentionCountRepositoryPurgeTest.java | 92 ++++- 11 files changed, 705 insertions(+), 200 deletions(-) delete mode 100644 archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/ArtifactFilenameFilter.java diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/AbstractRepositoryPurge.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/AbstractRepositoryPurge.java index 1f2a9e335..4c99fd64b 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/AbstractRepositoryPurge.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/AbstractRepositoryPurge.java @@ -37,9 +37,11 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.FilenameFilter; -import java.util.Collection; -import java.util.List; -import java.util.Set; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.stream.Collectors; /** * Base class for all repository purge tasks. @@ -47,7 +49,7 @@ import java.util.Set; public abstract class AbstractRepositoryPurge implements RepositoryPurge { - protected Logger log = LoggerFactory.getLogger( getClass() ); + protected Logger log = LoggerFactory.getLogger( getClass( ) ); protected final ManagedRepositoryContent repository; @@ -67,6 +69,116 @@ public abstract class AbstractRepositoryPurge this.listeners = listeners; } + /* + * We have to track namespace, project, project version, artifact version and classifier + * There is no metadata class that contains all these properties. + */ + class ArtifactInfo + { + final String namespace; + final String name; + final String projectVersion; + String version; + String classifier; + + ArtifactInfo( String namespace, String name, String projectVersion, String version ) + { + this.namespace = namespace; + this.name = name; + this.projectVersion = projectVersion; + this.version = version; + } + + ArtifactInfo( String namespace, String name, String projectVersion ) + { + this.namespace = namespace; + this.name = name; + this.projectVersion = projectVersion; + } + + /* + * Creates a info object without version and classifier + */ + ArtifactInfo projectVersionLevel( ) + { + return new ArtifactInfo( this.namespace, this.name, this.projectVersion ); + } + + public void setClassifier( String classifier ) + { + this.classifier = classifier; + } + + public String getNamespace( ) + { + return namespace; + } + + public String getName( ) + { + return name; + } + + public String getProjectVersion( ) + { + return projectVersion; + } + + public String getVersion( ) + { + return version; + } + + public String getClassifier( ) + { + return classifier; + } + + public boolean hasClassifier( ) + { + return classifier != null && !"".equals( classifier ); + } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) return true; + if ( o == null || getClass( ) != o.getClass( ) ) return false; + + ArtifactInfo that = (ArtifactInfo) o; + + if ( !namespace.equals( that.namespace ) ) return false; + if ( !name.equals( that.name ) ) return false; + if ( !projectVersion.equals( that.projectVersion ) ) return false; + if ( !( version != null ? version.equals( that.version ) : that.version == null ) ) return false; + return classifier != null ? classifier.equals( that.classifier ) : that.classifier == null; + } + + @Override + public int hashCode( ) + { + int result = namespace.hashCode( ); + result = 31 * result + name.hashCode( ); + result = 31 * result + projectVersion.hashCode( ); + result = 31 * result + ( version != null ? version.hashCode( ) : 0 ); + result = 31 * result + ( classifier != null ? classifier.hashCode( ) : 0 ); + return result; + } + + @Override + public String toString( ) + { + final StringBuilder sb = new StringBuilder( "ArtifactInfo{" ); + sb.append( "namespace='" ).append( namespace ).append( '\'' ); + sb.append( ", name='" ).append( name ).append( '\'' ); + sb.append( ", projectVersion='" ).append( projectVersion ).append( '\'' ); + sb.append( ", version='" ).append( version ).append( '\'' ); + sb.append( ", classifier='" ).append( classifier ).append( '\'' ); + sb.append( '}' ); + return sb.toString( ); + } + } + /** * Purge the repo. Update db and index of removed artifacts. * @@ -74,110 +186,192 @@ public abstract class AbstractRepositoryPurge */ protected void purge( Set references ) { - if ( references != null && !references.isEmpty() ) + if ( references != null && !references.isEmpty( ) ) { - MetadataRepository metadataRepository = repositorySession.getRepository(); + MetadataRepository metadataRepository = repositorySession.getRepository( ); + Map metaRemovalList = new HashMap<>( ); + Map> metaResolved = new HashMap<>( ); for ( ArtifactReference reference : references ) { - File artifactFile = repository.toFile( reference ); + String baseVersion = VersionUtil.getBaseVersion( reference.getVersion( ) ); + // Needed for tracking in the hashmap + String metaBaseId = reference.getGroupId( ) + "/" + reference.getArtifactId( ) + "/" + baseVersion; - for ( RepositoryListener listener : listeners ) + if ( !metaResolved.containsKey( metaBaseId ) ) { - listener.deleteArtifact( metadataRepository, repository.getId(), reference.getGroupId(), - reference.getArtifactId(), reference.getVersion(), - artifactFile.getName() ); + try + { + metaResolved.put( metaBaseId, metadataRepository.getArtifacts( repository.getId( ), reference.getGroupId( ), + reference.getArtifactId( ), baseVersion ) ); + } + catch ( MetadataResolutionException e ) + { + log.error( "Error during metadata retrieval {}: {}", metaBaseId, e.getMessage( ) ); + } } + Path artifactFile = repository.toFile( reference ).toPath( ); - // TODO: this needs to be logged - artifactFile.delete(); + for ( RepositoryListener listener : listeners ) + { + listener.deleteArtifact( metadataRepository, repository.getId( ), reference.getGroupId( ), + reference.getArtifactId( ), reference.getVersion( ), + artifactFile.getFileName( ).toString( ) ); + } try { - repository.deleteArtifact( reference ); + Files.delete( artifactFile ); + log.debug( "File deleted: {}", artifactFile.toAbsolutePath( ) ); } - catch ( ContentNotFoundException e ) + catch ( IOException e ) { - log.warn( "skip error deleting artifact {}: {}", reference, e.getMessage() ); + log.error( "Could not delete file {}: {}", artifactFile.toAbsolutePath( ), e.getMessage( ), e ); + continue; } - try { - metadataRepository.removeProjectVersion( repository.getId(), reference.getGroupId(), - reference.getArtifactId(), reference.getVersion() ); + repository.deleteArtifact( reference ); } - catch ( MetadataRepositoryException e ) + catch ( ContentNotFoundException e ) { - log.warn( "skip error removeProjectVersion artifact {}: {}", reference, e.getMessage() ); + log.warn( "skip error deleting artifact {}: {}", reference, e.getMessage( ) ); } - boolean snapshotVersion = VersionUtil.isSnapshot( reference.getVersion() ); + boolean snapshotVersion = VersionUtil.isSnapshot( reference.getVersion( ) ); - try + + // If this is a snapshot we have to search for artifacts with the same version. And remove all of them. + if ( snapshotVersion ) { - if ( snapshotVersion ) + Collection artifacts = + metaResolved.get( metaBaseId ); + if ( artifacts != null ) { - String baseVersion = VersionUtil.getBaseVersion( reference.getVersion() ); - Collection artifacts = - metadataRepository.getArtifacts( repository.getId(), reference.getGroupId(), - reference.getArtifactId(), baseVersion ); - if ( artifacts != null ) + // cleanup snapshots metadata + for ( ArtifactMetadata artifactMetadata : artifacts ) { - // cleanup snapshots metadata - for ( ArtifactMetadata artifactMetadata : artifacts ) + // Artifact metadata and reference version should match. + if ( artifactMetadata.getVersion( ).equals( reference.getVersion( ) ) ) { - - // TODO: mismatch between artifact (snapshot) version and project (base) version here - if ( artifactMetadata.getVersion().equals( reference.getVersion() ) ) + ArtifactInfo info = new ArtifactInfo( artifactMetadata.getNamespace( ), artifactMetadata.getProject( ), artifactMetadata.getProjectVersion( ), artifactMetadata.getVersion( ) ); + if ( StringUtils.isNotBlank( reference.getClassifier( ) ) ) { - if ( StringUtils.isNotBlank( reference.getClassifier() ) ) - { - - // cleanup facet which contains classifier information - MavenArtifactFacet mavenArtifactFacet = - (MavenArtifactFacet) artifactMetadata.getFacet( - MavenArtifactFacet.FACET_ID ); - - if ( StringUtils.equals( reference.getClassifier(), - mavenArtifactFacet.getClassifier() ) ) - { - artifactMetadata.removeFacet( MavenArtifactFacet.FACET_ID ); - String groupId = reference.getGroupId(), artifactId = - reference.getArtifactId(), - version = reference.getVersion(); - MavenArtifactFacet mavenArtifactFacetToCompare = new MavenArtifactFacet(); - mavenArtifactFacetToCompare.setClassifier( reference.getClassifier() ); - metadataRepository.removeArtifact( repository.getId(), groupId, artifactId, - version, mavenArtifactFacetToCompare ); - metadataRepository.save(); - } - - } - else - { - metadataRepository.removeArtifact( artifactMetadata, VersionUtil.getBaseVersion( - reference.getVersion() ) ); - } - + info.setClassifier( reference.getClassifier( ) ); + metaRemovalList.put( info, artifactMetadata ); + } + else + { + // metadataRepository.removeArtifact( artifactMetadata, baseVersion ); + metaRemovalList.put( info, artifactMetadata ); } } } } } - catch ( MetadataResolutionException e ) + else // otherwise we delete the artifact version { - log.warn( "skip error deleting metadata {}: {}", reference, e.getMessage() ); + ArtifactInfo info = new ArtifactInfo( reference.getGroupId( ), reference.getArtifactId( ), baseVersion, reference.getVersion( ) ); + for ( ArtifactMetadata metadata : metaResolved.get( metaBaseId ) ) + { + metaRemovalList.put( info, metadata ); + } } - catch ( MetadataRepositoryException e ) + triggerAuditEvent( repository.getRepository( ).getId( ), ArtifactReference.toKey( reference ), + AuditEvent.PURGE_ARTIFACT ); + purgeSupportFiles( artifactFile ); + } + purgeMetadata( metadataRepository, metaRemovalList ); + repositorySession.save( ); + + } + } + + /* + * Purges the metadata. First removes the artifacts. After that empty versions will be removed. + */ + private void purgeMetadata( MetadataRepository metadataRepository, Map dataList ) + { + Set projectLevelMetadata = new HashSet<>( ); + for ( Map.Entry infoEntry : dataList.entrySet( ) ) + { + ArtifactInfo info = infoEntry.getKey( ); + try + { + removeArtifact( metadataRepository, info, infoEntry.getValue( ) ); + log.debug( "Removed artifact from MetadataRepository {}", info ); + } + catch ( MetadataRepositoryException e ) + { + log.error( "Could not remove artifact from MetadataRepository {}: {}", info, e.getMessage( ), e ); + } + projectLevelMetadata.add( info.projectVersionLevel( ) ); + } + metadataRepository.save( ); + Collection artifacts = null; + // Get remaining artifacts and remove project if empty + for ( ArtifactInfo info : projectLevelMetadata ) + { + try + { + artifacts = metadataRepository.getArtifacts( repository.getId( ), info.getNamespace( ), info.getName( ), + info.getProjectVersion( ) ); + if ( artifacts.size( ) == 0 ) { - log.warn( "skip error deleting metadata {}: {}", reference, e.getMessage() ); + metadataRepository.removeProjectVersion( repository.getId( ), info.getNamespace( ), + info.getName( ), info.getProjectVersion( ) ); + log.debug( "Removed project version from MetadataRepository {}", info ); } + } + catch ( MetadataResolutionException | MetadataRepositoryException e ) + { + log.error( "Could not remove project version from MetadataRepository {}: {}", info, e.getMessage( ), e ); + } + } + metadataRepository.save( ); - repositorySession.save(); + } - triggerAuditEvent( repository.getRepository().getId(), ArtifactReference.toKey( reference ), - AuditEvent.PURGE_ARTIFACT ); - purgeSupportFiles( artifactFile ); + /* + * Removes the artifact from the metadataRepository. If a classifier is set, the facet will be removed. + */ + private void removeArtifact( MetadataRepository metadataRepository, ArtifactInfo artifactInfo, ArtifactMetadata artifactMetadata ) throws MetadataRepositoryException + { + if ( artifactInfo.hasClassifier( ) ) + { + // cleanup facet which contains classifier information + MavenArtifactFacet mavenArtifactFacet = + (MavenArtifactFacet) artifactMetadata.getFacet( + MavenArtifactFacet.FACET_ID ); + + if ( StringUtils.equals( artifactInfo.classifier, + mavenArtifactFacet.getClassifier( ) ) ) + { + artifactMetadata.removeFacet( MavenArtifactFacet.FACET_ID ); + String groupId = artifactInfo.getNamespace( ), artifactId = + artifactInfo.getName( ), + version = artifactInfo.getProjectVersion( ); + MavenArtifactFacet mavenArtifactFacetToCompare = new MavenArtifactFacet( ); + mavenArtifactFacetToCompare.setClassifier( artifactInfo.getClassifier( ) ); + metadataRepository.removeArtifact( repository.getId( ), groupId, artifactId, + version, mavenArtifactFacetToCompare ); + metadataRepository.save( ); } } + else + { + metadataRepository.removeArtifact( artifactMetadata, artifactInfo.getProjectVersion( ) ); + } + } + + private void deleteSilently(Path path) { + try + { + Files.deleteIfExists( path ); + triggerAuditEvent( repository.getRepository( ).getId( ), path.toString(), AuditEvent.PURGE_FILE ); + } + catch ( IOException e ) + { + log.error("Error occured during file deletion {}: {} ",path,e.getMessage(), e); + } } /** @@ -190,29 +384,28 @@ public abstract class AbstractRepositoryPurge * * @param artifactFile the file to base off of. */ - private void purgeSupportFiles( File artifactFile ) + private void purgeSupportFiles( Path artifactFile ) { - File parentDir = artifactFile.getParentFile(); + Path parentDir = artifactFile.getParent(); - if ( !parentDir.exists() ) + if ( !Files.exists(parentDir) ) { return; } - FilenameFilter filter = new ArtifactFilenameFilter( artifactFile.getName() ); + final String artifactName = artifactFile.getFileName().toString(); - File[] files = parentDir.listFiles( filter ); - - for ( File file : files ) + try { - if ( file.exists() && file.isFile() ) - { - String fileName = file.getName(); - file.delete(); - // TODO: log that it was deleted - triggerAuditEvent( repository.getRepository().getId(), fileName, AuditEvent.PURGE_FILE ); - } + Files.find(parentDir, 3, + ( path, basicFileAttributes ) -> path.getFileName().toString().startsWith(artifactName) + && Files.isRegularFile( path ) ).forEach( this::deleteSilently ); } + catch ( IOException e ) + { + log.error("Purge of support files failed {}: {}", artifactFile, e.getMessage(), e); + } + } private void triggerAuditEvent( String repoId, String resource, String action ) diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/ArtifactFilenameFilter.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/ArtifactFilenameFilter.java deleted file mode 100644 index f1f2c6216..000000000 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/ArtifactFilenameFilter.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.apache.archiva.consumers.core.repository; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.io.FilenameFilter; -import java.io.File; - -/** - * Filename filter for getting all the files related to a specific artifact. - * - */ -public class ArtifactFilenameFilter - implements FilenameFilter -{ - private String filename; - - public ArtifactFilenameFilter() - { - // no op - } - - public ArtifactFilenameFilter( String filename ) - { - this.filename = filename; - } - - @Override - public boolean accept( File dir, String name ) - { - return ( name.startsWith( filename ) ); - } -} diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/CleanupReleasedSnapshotsRepositoryPurge.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/CleanupReleasedSnapshotsRepositoryPurge.java index a91cc5282..777d1e676 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/CleanupReleasedSnapshotsRepositoryPurge.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/CleanupReleasedSnapshotsRepositoryPurge.java @@ -23,6 +23,7 @@ import org.apache.archiva.admin.model.RepositoryAdminException; import org.apache.archiva.admin.model.beans.ManagedRepository; import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; import org.apache.archiva.metadata.repository.MetadataRepository; +import org.apache.archiva.metadata.repository.MetadataRepositoryException; import org.apache.archiva.metadata.repository.RepositorySession; import org.apache.archiva.repository.events.RepositoryListener; import org.apache.archiva.common.utils.VersionComparator; @@ -159,13 +160,14 @@ public class CleanupReleasedSnapshotsRepositoryPurge versionRef.setVersion( artifactRef.getVersion() ); repository.deleteVersion( versionRef ); - // FIXME: looks incomplete, might not delete related metadata? for ( RepositoryListener listener : listeners ) { listener.deleteArtifact( metadataRepository, repository.getId(), artifactRef.getGroupId(), artifactRef.getArtifactId(), artifactRef.getVersion(), artifactFile.getName() ); } + metadataRepository.removeProjectVersion( repository.getId(), artifactRef.getGroupId(), + artifactRef.getArtifactId(), artifactRef.getVersion()); needsMetadataUpdate = true; } @@ -186,6 +188,10 @@ public class CleanupReleasedSnapshotsRepositoryPurge { throw new RepositoryPurgeException( e.getMessage(), e ); } + catch ( MetadataRepositoryException e ) + { + log.error("Could not remove metadata during cleanup of released snapshots of {}", path, e); + } } private void updateMetadata( ArtifactReference artifact ) diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurge.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurge.java index da0591c85..6ba811bea 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurge.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurge.java @@ -33,12 +33,7 @@ import org.apache.archiva.repository.layout.LayoutException; import java.io.File; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.regex.Matcher; /** @@ -99,6 +94,7 @@ public class DaysOldRepositoryPurge int countToPurge = versions.size() - retentionCount; + Set artifactsToDelete = new HashSet<>(); for ( String version : versions ) { if ( countToPurge-- <= 0 ) @@ -117,7 +113,7 @@ public class DaysOldRepositoryPurge { if ( newArtifactFile.lastModified() < olderThanThisDate.getTimeInMillis() ) { - doPurgeAllRelated( newArtifactReference ); + artifactsToDelete.addAll(repository.getRelatedArtifacts(newArtifactReference) ); } } // Is this a timestamp snapshot "1.0-20070822.123456-42" ? @@ -127,10 +123,11 @@ public class DaysOldRepositoryPurge if ( timestampCal.getTimeInMillis() < olderThanThisDate.getTimeInMillis() ) { - doPurgeAllRelated( newArtifactReference ); + artifactsToDelete.addAll( repository.getRelatedArtifacts(newArtifactReference)); } } } + purge(artifactsToDelete); } catch ( ContentNotFoundException e ) { @@ -175,17 +172,4 @@ public class DaysOldRepositoryPurge return null; } - private void doPurgeAllRelated( ArtifactReference reference ) - { - try - { - Set related = repository.getRelatedArtifacts( reference ); - purge( related ); - } - catch ( ContentNotFoundException e ) - { - // Nothing to do here - it means the repository would have been constructed incorrectly - log.debug( e.getMessage(), e ); - } - } } diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RepositoryPurgeConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RepositoryPurgeConsumer.java index 41e4761f9..af71b9eb2 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RepositoryPurgeConsumer.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RepositoryPurgeConsumer.java @@ -102,9 +102,6 @@ public class RepositoryPurgeConsumer @Autowired(required = false) private List listeners = Collections.emptyList(); - /** - * FIXME: this could be multiple implementations and needs to be configured. - */ @Inject private RepositorySessionFactory repositorySessionFactory; @@ -189,7 +186,6 @@ public class RepositoryPurgeConsumer { cleanUp.process( path ); } - repoPurge.process( path ); } catch ( RepositoryPurgeException rpe ) diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurge.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurge.java index 5cbcaa406..29d1e552c 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurge.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurge.java @@ -32,6 +32,7 @@ import org.apache.archiva.repository.layout.LayoutException; import java.io.File; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -78,21 +79,22 @@ public class RetentionCountRepositoryPurge if ( retentionCount > versions.size() ) { + log.trace("No deletion, because retention count is higher than actual number of artifacts."); // Done. nothing to do here. skip it. return; } int countToPurge = versions.size() - retentionCount; - + Set artifactsToDelete = new HashSet<>(); for ( String version : versions ) { if ( countToPurge-- <= 0 ) { break; } - - doPurgeAllRelated( artifact, version ); + artifactsToDelete.addAll(repository.getRelatedArtifacts( getNewArtifactReference( artifact, version) )); } + purge(artifactsToDelete); } } catch ( LayoutException le ) @@ -101,12 +103,14 @@ public class RetentionCountRepositoryPurge } catch ( ContentNotFoundException e ) { - // Nothing to do here. - // TODO: Log this condition? + log.error("Repostory artifact not found {}", path); } } - private void doPurgeAllRelated( ArtifactReference reference, String version ) + /* + * Returns a new artifact reference with different version + */ + private ArtifactReference getNewArtifactReference( ArtifactReference reference, String version ) throws LayoutException { ArtifactReference artifact = new ArtifactReference(); @@ -115,16 +119,7 @@ public class RetentionCountRepositoryPurge artifact.setVersion( version ); artifact.setClassifier( reference.getClassifier() ); artifact.setType( reference.getType() ); + return artifact; - try - { - Set related = repository.getRelatedArtifacts( artifact ); - purge( related ); - } - catch ( ContentNotFoundException e ) - { - // Nothing to do here. - // TODO: Log this? - } } } diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/AbstractRepositoryPurgeTest.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/AbstractRepositoryPurgeTest.java index 9412ac73c..764596684 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/AbstractRepositoryPurgeTest.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/AbstractRepositoryPurgeTest.java @@ -21,8 +21,11 @@ package org.apache.archiva.consumers.core.repository; import org.apache.archiva.admin.model.beans.ManagedRepository; import org.apache.archiva.common.plexusbridge.PlexusSisuBridge; +import org.apache.archiva.common.utils.VersionUtil; +import org.apache.archiva.metadata.model.ArtifactMetadata; import org.apache.archiva.metadata.repository.MetadataRepository; import org.apache.archiva.metadata.repository.RepositorySession; +import org.apache.archiva.metadata.repository.storage.maven2.Maven2RepositoryPathTranslator; import org.apache.archiva.repository.ManagedRepositoryContent; import org.apache.archiva.repository.events.RepositoryListener; import org.apache.archiva.test.utils.ArchivaSpringJUnit4ClassRunner; @@ -41,6 +44,14 @@ import org.springframework.test.context.ContextConfiguration; import javax.inject.Inject; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -194,6 +205,10 @@ public abstract class AbstractRepositoryPurgeTest return new File( "target/test-" + getName() + "/" + TEST_REPO_ID ); } + protected Path getTestRepoRootPath() { + return Paths.get("target/test-"+getName()+"/"+TEST_REPO_ID); + } + protected String prepareTestRepos() throws Exception { @@ -218,4 +233,14 @@ public abstract class AbstractRepositoryPurgeTest { return StringUtils.substringAfterLast( getClass().getName(), "." ); } + + protected List getArtifactMetadataFromDir( String repoId, String projectName, Path repoDir, Path vDir ) throws IOException + { + Maven2RepositoryPathTranslator translator = new Maven2RepositoryPathTranslator( new ArrayList<>( ) ); + return Files.find(vDir, 1, + (path, basicFileAttributes) -> basicFileAttributes.isRegularFile() && path.getFileName().toString().startsWith(projectName)) + .map( path -> + translator.getArtifactForPath( repoId, repoDir.relativize( path ).toString() ) + ).collect( Collectors.toList()); + } } diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/CleanupReleasedSnapshotsRepositoryPurgeTest.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/CleanupReleasedSnapshotsRepositoryPurgeTest.java index 5909657db..697ec249f 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/CleanupReleasedSnapshotsRepositoryPurgeTest.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/CleanupReleasedSnapshotsRepositoryPurgeTest.java @@ -22,6 +22,8 @@ package org.apache.archiva.consumers.core.repository; import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; import org.apache.archiva.admin.repository.managed.DefaultManagedRepositoryAdmin; import org.apache.archiva.configuration.ArchivaConfiguration; +import org.apache.archiva.metadata.model.ArtifactMetadata; +import org.apache.archiva.metadata.model.MetadataFacet; import org.apache.archiva.repository.RepositoryContentFactory; import org.apache.archiva.repository.events.RepositoryListener; import org.apache.archiva.repository.metadata.MetadataTools; @@ -30,15 +32,21 @@ import org.custommonkey.xmlunit.XMLAssert; import org.easymock.EasyMock; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.springframework.test.context.ContextConfiguration; import javax.inject.Inject; import java.io.File; import java.nio.charset.Charset; +import java.nio.file.Path; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.*; /** @@ -92,7 +100,7 @@ public class CleanupReleasedSnapshotsRepositoryPurgeTest removeMavenIndexes(); } - //@Test + @Test public void testReleasedSnapshotsExistsInSameRepo() throws Exception { @@ -101,17 +109,35 @@ public class CleanupReleasedSnapshotsRepositoryPurgeTest getRepoConfiguration( TEST_REPO_ID, TEST_REPO_NAME ), false, null ); String repoRoot = prepareTestRepos(); + String projectNs = "org.apache.maven.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "maven-plugin-plugin"; + String projectVersion = "2.3-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("2.3-SNAPSHOT"); // test listeners for the correct artifacts listener.deleteArtifact( metadataRepository, getRepository().getId(), "org.apache.maven.plugins", "maven-plugin-plugin", "2.3-SNAPSHOT", "maven-plugin-plugin-2.3-SNAPSHOT.jar" ); listenerControl.replay(); + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + + repoPurge.process( PATH_TO_RELEASED_SNAPSHOT_IN_SAME_REPO ); listenerControl.verify(); - String projectRoot = repoRoot + "/org/apache/maven/plugins/maven-plugin-plugin"; + // Verify the metadataRepository invocations + // complete snapshot version removal for released + verify(metadataRepository, times(1)).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq("2.3")); // check if the snapshot was removed assertDeleted( projectRoot + "/2.3-SNAPSHOT" ); @@ -149,7 +175,7 @@ public class CleanupReleasedSnapshotsRepositoryPurgeTest XMLAssert.assertXpathEvaluatesTo( "20070315032817", "//metadata/versioning/lastUpdated", metadataXml ); } - //@Test + @Test public void testNonArtifactFile() throws Exception { @@ -179,7 +205,7 @@ public class CleanupReleasedSnapshotsRepositoryPurgeTest assertTrue( file.exists() ); } - //@Test + @Test public void testReleasedSnapshotsExistsInDifferentRepo() throws Exception { @@ -192,6 +218,18 @@ public class CleanupReleasedSnapshotsRepositoryPurgeTest getRepoConfiguration( RELEASES_TEST_REPO_ID, RELEASES_TEST_REPO_NAME ), false, null ); String repoRoot = prepareTestRepos(); + String projectNs = "org.apache.archiva"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "released-artifact-in-diff-repo"; + String projectVersion = "1.0-SNAPSHOT"; + String releaseVersion = "1.0"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + Path releaseDir = repo.getParent().resolve(RELEASES_TEST_REPO_ID).resolve(projectPath).resolve(projectName).resolve(releaseVersion); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("1.0-SNAPSHOT"); + // test listeners for the correct artifacts listener.deleteArtifact( metadataRepository, getRepository().getId(), "org.apache.archiva", @@ -199,11 +237,25 @@ public class CleanupReleasedSnapshotsRepositoryPurgeTest "released-artifact-in-diff-repo-1.0-SNAPSHOT.jar" ); listenerControl.replay(); + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + + List ml2 = getArtifactMetadataFromDir(RELEASES_TEST_REPO_ID , projectName, repo.getParent(), releaseDir ); + when(metadataRepository.getArtifacts(RELEASES_TEST_REPO_ID, projectNs, + projectName, releaseVersion)).thenReturn(ml2); + + repoPurge.process( PATH_TO_RELEASED_SNAPSHOT_IN_DIFF_REPO ); listenerControl.verify(); - String projectRoot = repoRoot + "/org/apache/archiva/released-artifact-in-diff-repo"; + // Verify the metadataRepository invocations + // Complete version removal for cleanup + verify(metadataRepository, times(1)).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + verify(metadataRepository, never()).removeProjectVersion(eq(RELEASES_TEST_REPO_ID), eq(projectNs), eq(projectName), eq(releaseVersion)); + // check if the snapshot was removed assertDeleted( projectRoot + "/1.0-SNAPSHOT" ); @@ -241,15 +293,44 @@ public class CleanupReleasedSnapshotsRepositoryPurgeTest getRepoConfiguration( TEST_REPO_ID, TEST_REPO_NAME ), false, null ); String repoRoot = prepareTestRepos(); + String projectNs = "org.apache.maven.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "maven-source-plugin"; + String projectVersion = "2.0.2"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + Path vDir2 = repo.resolve(projectPath).resolve(projectName).resolve("2.0.3-SNAPSHOT"); + Path vDir3 = repo.resolve(projectPath).resolve(projectName).resolve("2.0.4-SNAPSHOT"); // test listeners for the correct artifacts - no deletions listenerControl.replay(); + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + List m2 = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir2 ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, "2.0.3-SNAPSHOT")).thenReturn(ml); + List m3 = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir3 ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, "2.0.4-SNAPSHOT")).thenReturn(ml); + + repoPurge.process( CleanupReleasedSnapshotsRepositoryPurgeTest.PATH_TO_HIGHER_SNAPSHOT_EXISTS_IN_SAME_REPO ); listenerControl.verify(); - String projectRoot = repoRoot + "/org/apache/maven/plugins/maven-source-plugin"; + // Verify the metadataRepository invocations + // No removal + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq("2.0.3-SNAPSHOT")); + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq("2.0.4-SNAPSHOT")); + verify(metadataRepository, never()).removeArtifact(any(ArtifactMetadata.class), any(String.class)); + verify(metadataRepository, never()).removeArtifact(any(String.class), any(String.class), any(String.class), any(String.class), any( MetadataFacet.class)); + + // check if the snapshot was not removed assertExists( projectRoot + "/2.0.3-SNAPSHOT" ); diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurgeTest.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurgeTest.java index 21b03da22..fbc25fdef 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurgeTest.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurgeTest.java @@ -20,15 +20,25 @@ package org.apache.archiva.consumers.core.repository; */ import java.io.File; +import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; + import org.apache.archiva.admin.model.beans.ManagedRepository; +import org.apache.archiva.metadata.model.ArtifactMetadata; import org.apache.archiva.repository.events.RepositoryListener; import org.apache.commons.lang.time.DateUtils; import org.junit.After; import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.*; /** */ @@ -66,10 +76,18 @@ public class DaysOldRepositoryPurgeTest Collections.singletonList( listener ) ); String repoRoot = prepareTestRepos(); - - String projectRoot = repoRoot + "/org/apache/maven/plugins/maven-install-plugin"; - - setLastModified( projectRoot + "/2.2-SNAPSHOT/", OLD_TIMESTAMP ); + String projectNs = "org.apache.maven.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "maven-install-plugin"; + String projectVersion = "2.2-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("2.2-SNAPSHOT"); + deletedVersions.add("2.2-20061118.060401-2"); + + setLastModified( projectRoot + "/" + projectVersion + "/", OLD_TIMESTAMP ); // test listeners for the correct artifacts listener.deleteArtifact( metadataRepository, getRepository().getId(), "org.apache.maven.plugins", @@ -84,10 +102,26 @@ public class DaysOldRepositoryPurgeTest "maven-install-plugin-2.2-20061118.060401-2.pom" ); listenerControl.replay(); + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + repoPurge.process( PATH_TO_BY_DAYS_OLD_ARTIFACT ); listenerControl.verify(); + // Verify the metadataRepository invocations + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, times(2)).removeArtifact(metadataArg.capture(), eq(projectVersion)); + List metaL = metadataArg.getAllValues(); + for (ArtifactMetadata meta : metaL) { + assertTrue(meta.getId().startsWith(projectName)); + assertTrue(deletedVersions.contains(meta.getVersion())); + } + + assertDeleted( projectRoot + "/2.2-SNAPSHOT/maven-install-plugin-2.2-SNAPSHOT.jar" ); assertDeleted( projectRoot + "/2.2-SNAPSHOT/maven-install-plugin-2.2-SNAPSHOT.jar.md5" ); assertDeleted( projectRoot + "/2.2-SNAPSHOT/maven-install-plugin-2.2-SNAPSHOT.jar.sha1" ); @@ -128,10 +162,17 @@ public class DaysOldRepositoryPurgeTest repoConfiguration.getRetentionCount(), repositorySession, listeners ); String repoRoot = prepareTestRepos(); - - String projectRoot = repoRoot + "/org/apache/maven/plugins/maven-assembly-plugin"; - - setLastModified( projectRoot + "/1.1.2-SNAPSHOT/", OLD_TIMESTAMP ); + String projectNs = "org.apache.maven.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "maven-assembly-plugin"; + String projectVersion = "1.1.2-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("1.1.2-20070427.065136-1"); + + setLastModified( projectRoot + "/" + projectVersion + "/", OLD_TIMESTAMP ); // test listeners for the correct artifacts listener.deleteArtifact( metadataRepository, getRepository().getId(), "org.apache.maven.plugins", @@ -142,10 +183,27 @@ public class DaysOldRepositoryPurgeTest "maven-assembly-plugin-1.1.2-20070427.065136-1.pom" ); listenerControl.replay(); + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + + repoPurge.process( PATH_TO_TEST_ORDER_OF_DELETION ); listenerControl.verify(); + // Verify the metadataRepository invocations + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, times(deletedVersions.size())).removeArtifact(metadataArg.capture(), eq(projectVersion)); + List metaL = metadataArg.getAllValues(); + for (ArtifactMetadata meta : metaL) { + assertTrue(meta.getId().startsWith(projectName)); + assertTrue(deletedVersions.contains(meta.getVersion())); + } + + assertDeleted( projectRoot + "/1.1.2-SNAPSHOT/maven-assembly-plugin-1.1.2-20070427.065136-1.jar" ); assertDeleted( projectRoot + "/1.1.2-SNAPSHOT/maven-assembly-plugin-1.1.2-20070427.065136-1.jar.sha1" ); assertDeleted( projectRoot + "/1.1.2-SNAPSHOT/maven-assembly-plugin-1.1.2-20070427.065136-1.jar.md5" ); @@ -179,8 +237,18 @@ public class DaysOldRepositoryPurgeTest repoConfiguration.getRetentionCount(), repositorySession, listeners ); String repoRoot = prepareTestRepos(); + String projectNs = "org.codehaus.plexus"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "plexus-utils"; + String projectVersion = "1.4.3-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("1.4.3-20070113.163208-4"); + - String versionRoot = repoRoot + "/org/codehaus/plexus/plexus-utils/1.4.3-SNAPSHOT"; + String versionRoot = projectRoot + "/"+ projectVersion; Calendar currentDate = Calendar.getInstance( DateUtils.UTC_TIME_ZONE ); setLastModified( versionRoot, currentDate.getTimeInMillis() ); @@ -209,10 +277,27 @@ public class DaysOldRepositoryPurgeTest "1.4.3-20070113.163208-4", "plexus-utils-1.4.3-20070113.163208-4.pom" ); listenerControl.replay(); + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + + repoPurge.process( PATH_TO_BY_DAYS_OLD_METADATA_DRIVEN_ARTIFACT ); listenerControl.verify(); + // Verify the metadataRepository invocations + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, times(deletedVersions.size())).removeArtifact(metadataArg.capture(), eq(projectVersion)); + List metaL = metadataArg.getAllValues(); + for (ArtifactMetadata meta : metaL) { + assertTrue(meta.getId().startsWith(projectName)); + assertTrue(deletedVersions.contains(meta.getVersion())); + } + + // this should be deleted since the filename version (timestamp) is older than // 100 days even if the last modified date was <100 days ago assertDeleted( versionRoot + "/plexus-utils-1.4.3-20070113.163208-4.jar" ); diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/RepositoryPurgeConsumerTest.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/RepositoryPurgeConsumerTest.java index 74e4631e3..6d5c33b40 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/RepositoryPurgeConsumerTest.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/RepositoryPurgeConsumerTest.java @@ -30,19 +30,38 @@ import org.apache.archiva.configuration.FileType; import org.apache.archiva.configuration.FileTypes; import org.apache.archiva.consumers.KnownRepositoryContentConsumer; import org.apache.archiva.consumers.functors.ConsumerWantsFilePredicate; +import org.apache.archiva.metadata.model.ArtifactMetadata; +import org.apache.archiva.metadata.model.MetadataFacet; +import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet; import org.apache.archiva.mock.MockRepositorySessionFactory; import org.apache.commons.io.FileUtils; import org.custommonkey.xmlunit.XMLAssert; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.springframework.test.context.ContextConfiguration; +import sun.awt.image.ImageWatched; import java.io.File; +import java.io.IOException; import java.nio.charset.Charset; - +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.sun.imageio.plugins.jpeg.JPEG.version; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.*; /** */ @@ -152,10 +171,37 @@ public class RepositoryPurgeConsumerTest repoPurgeConsumer.beginScan( repoConfiguration, null ); String repoRoot = prepareTestRepos(); + String projectNs = "org.jruby.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "jruby-rake-plugin"; + String projectVersion = "1.0RC1-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + String versionRoot = projectRoot + "/" + projectVersion; + + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + + // Provide the metadata list + List ml = getArtifactMetadataFromDir( TEST_REPO_ID, projectName, repo, vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("1.0RC1-20070504.153317-1"); + deletedVersions.add("1.0RC1-20070504.160758-2"); repoPurgeConsumer.processFile( PATH_TO_BY_RETENTION_COUNT_ARTIFACT ); - String versionRoot = repoRoot + "/org/jruby/plugins/jruby-rake-plugin/1.0RC1-SNAPSHOT"; + // Verify the metadataRepository invocations + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, times(2)).removeArtifact(metadataArg.capture(), eq(projectVersion)); + List metaL = metadataArg.getAllValues(); + for (ArtifactMetadata meta : metaL) { + assertTrue(meta.getId().startsWith(projectName)); + assertTrue(deletedVersions.contains(meta.getVersion())); + } + + // assert if removed from repo assertDeleted( versionRoot + "/jruby-rake-plugin-1.0RC1-20070504.153317-1.jar" ); @@ -247,12 +293,38 @@ public class RepositoryPurgeConsumerTest repoPurgeConsumer.beginScan( repoConfiguration, null ); String repoRoot = prepareTestRepos(); - String projectRoot = repoRoot + "/org/apache/maven/plugins/maven-install-plugin"; + String projectNs = "org.apache.maven.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "maven-install-plugin"; + String projectVersion = "2.2-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + + setLastModified( projectRoot + "/"+projectVersion); + - setLastModified( projectRoot + "/2.2-SNAPSHOT" ); + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + + // Provide the metadata list + List ml = getArtifactMetadataFromDir( TEST_REPO_ID, projectName, repo, vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("2.2-SNAPSHOT"); + deletedVersions.add("2.2-20061118.060401-2"); repoPurgeConsumer.processFile( PATH_TO_BY_DAYS_OLD_ARTIFACT ); + // Verify the metadataRepository invocations + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, times(2)).removeArtifact(metadataArg.capture(), eq(projectVersion)); + List metaL = metadataArg.getAllValues(); + for (ArtifactMetadata meta : metaL) { + assertTrue(meta.getId().startsWith(projectName)); + assertTrue(deletedVersions.contains(meta.getVersion())); + } + assertDeleted( projectRoot + "/2.2-SNAPSHOT/maven-install-plugin-2.2-SNAPSHOT.jar" ); assertDeleted( projectRoot + "/2.2-SNAPSHOT/maven-install-plugin-2.2-SNAPSHOT.jar.md5" ); assertDeleted( projectRoot + "/2.2-SNAPSHOT/maven-install-plugin-2.2-SNAPSHOT.jar.sha1" ); @@ -305,12 +377,29 @@ public class RepositoryPurgeConsumerTest repoPurgeConsumer.beginScan( repoConfiguration, null ); String repoRoot = prepareTestRepos(); + String projectNs = "org.apache.maven.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "maven-plugin-plugin"; + String projectVersion = "2.3-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + + // Provide the metadata list + List ml = getArtifactMetadataFromDir( TEST_REPO_ID, projectName, repo, vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); repoPurgeConsumer.processFile( CleanupReleasedSnapshotsRepositoryPurgeTest.PATH_TO_RELEASED_SNAPSHOT_IN_SAME_REPO ); + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, never()).removeArtifact(any(), any()); + verify(metadataRepository, never()).removeArtifact( any(), any(), any(), any(), any(MetadataFacet.class) ); + // check if the snapshot wasn't removed - String projectRoot = repoRoot + "/org/apache/maven/plugins/maven-plugin-plugin"; assertExists( projectRoot + "/2.3-SNAPSHOT" ); assertExists( projectRoot + "/2.3-SNAPSHOT/maven-plugin-plugin-2.3-SNAPSHOT.jar" ); @@ -350,11 +439,25 @@ public class RepositoryPurgeConsumerTest repoPurgeConsumer.beginScan( repoConfiguration, null ); String repoRoot = prepareTestRepos(); + String projectNs = "org.apache.maven.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "maven-plugin-plugin"; + String projectVersion = "2.3-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); repoPurgeConsumer.processFile( CleanupReleasedSnapshotsRepositoryPurgeTest.PATH_TO_RELEASED_SNAPSHOT_IN_SAME_REPO ); - String projectRoot = repoRoot + "/org/apache/maven/plugins/maven-plugin-plugin"; + verify(metadataRepository, times(1)).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, never()).removeArtifact(any(), any()); // check if the snapshot was removed assertDeleted( projectRoot + "/2.3-SNAPSHOT" ); diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurgeTest.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurgeTest.java index f7e21f101..ca7685b02 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurgeTest.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurgeTest.java @@ -19,13 +19,23 @@ package org.apache.archiva.consumers.core.repository; * under the License. */ +import java.nio.file.Path; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; + import org.apache.archiva.admin.model.beans.ManagedRepository; +import org.apache.archiva.metadata.model.ArtifactMetadata; import org.apache.archiva.repository.events.RepositoryListener; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.*; /** * Test RetentionsCountRepositoryPurgeTest @@ -62,6 +72,17 @@ public class RetentionCountRepositoryPurgeTest throws Exception { String repoRoot = prepareTestRepos(); + String projectNs = "org.jruby.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "jruby-rake-plugin"; + String projectVersion = "1.0RC1-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("1.0RC1-20070504.153317-1"); + deletedVersions.add("1.0RC1-20070504.160758-2"); + String versionRoot = projectRoot + "/" + projectVersion; // test listeners for the correct artifacts listener.deleteArtifact( metadataRepository, getRepository().getId(), "org.jruby.plugins", "jruby-rake-plugin", @@ -87,11 +108,26 @@ public class RetentionCountRepositoryPurgeTest "1.0RC1-20070504.160758-2", "jruby-rake-plugin-1.0RC1-20070504.160758-2.pom" ); listenerControl.replay(); + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + + repoPurge.process( PATH_TO_BY_RETENTION_COUNT_ARTIFACT ); listenerControl.verify(); - String versionRoot = repoRoot + "/org/jruby/plugins/jruby-rake-plugin/1.0RC1-SNAPSHOT"; + // Verify the metadataRepository invocations + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, times(deletedVersions.size())).removeArtifact(metadataArg.capture(), eq(projectVersion)); + List metaL = metadataArg.getAllValues(); + for (ArtifactMetadata meta : metaL) { + assertTrue(meta.getId().startsWith(projectName)); + assertTrue(deletedVersions.contains(meta.getVersion())); + } + // assert if removed from repo assertDeleted( versionRoot + "/jruby-rake-plugin-1.0RC1-20070504.153317-1.jar" ); @@ -132,6 +168,17 @@ public class RetentionCountRepositoryPurgeTest throws Exception { String repoRoot = prepareTestRepos(); + String projectNs = "org.codehaus.castor"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "castor-anttasks"; + String projectVersion = "1.1.2-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("1.1.2-20070427.065136-1"); + String versionRoot = projectRoot + "/" + projectVersion; + // test listeners for the correct artifacts listener.deleteArtifact( metadataRepository, getRepository().getId(), "org.codehaus.castor", "castor-anttasks", @@ -140,11 +187,25 @@ public class RetentionCountRepositoryPurgeTest "1.1.2-20070427.065136-1", "castor-anttasks-1.1.2-20070427.065136-1.pom" ); listenerControl.replay(); + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + repoPurge.process( PATH_TO_BY_RETENTION_COUNT_POM ); listenerControl.verify(); - String versionRoot = repoRoot + "/org/codehaus/castor/castor-anttasks/1.1.2-SNAPSHOT"; + // Verify the metadataRepository invocations + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, times(deletedVersions.size())).removeArtifact(metadataArg.capture(), eq(projectVersion)); + List metaL = metadataArg.getAllValues(); + for (ArtifactMetadata meta : metaL) { + assertTrue(meta.getId().startsWith(projectName)); + assertTrue(deletedVersions.contains(meta.getVersion())); + } + // assert if removed from repo assertDeleted( versionRoot + "/castor-anttasks-1.1.2-20070427.065136-1.jar" ); @@ -181,6 +242,17 @@ public class RetentionCountRepositoryPurgeTest throws Exception { String repoRoot = prepareTestRepos(); + String projectNs = "org.apache.maven.plugins"; + String projectPath = projectNs.replaceAll("\\.","/"); + String projectName = "maven-assembly-plugin"; + String projectVersion = "1.1.2-SNAPSHOT"; + String projectRoot = repoRoot + "/" + projectPath+"/"+projectName; + Path repo = getTestRepoRootPath(); + Path vDir = repo.resolve(projectPath).resolve(projectName).resolve(projectVersion); + Set deletedVersions = new HashSet<>(); + deletedVersions.add("1.1.2-20070427.065136-1"); + String versionRoot = projectRoot + "/" + projectVersion; + // test listeners for the correct artifacts listener.deleteArtifact( metadataRepository, getRepository().getId(), "org.apache.maven.plugins", @@ -191,11 +263,25 @@ public class RetentionCountRepositoryPurgeTest "maven-assembly-plugin-1.1.2-20070427.065136-1.pom" ); listenerControl.replay(); + // Provide the metadata list + List ml = getArtifactMetadataFromDir(TEST_REPO_ID , projectName, repo.getParent(), vDir ); + when(metadataRepository.getArtifacts(TEST_REPO_ID, projectNs, + projectName, projectVersion)).thenReturn(ml); + repoPurge.process( PATH_TO_TEST_ORDER_OF_DELETION ); listenerControl.verify(); - String versionRoot = repoRoot + "/org/apache/maven/plugins/maven-assembly-plugin/1.1.2-SNAPSHOT"; + // Verify the metadataRepository invocations + verify(metadataRepository, never()).removeProjectVersion(eq(TEST_REPO_ID), eq(projectNs), eq(projectName), eq(projectVersion)); + ArgumentCaptor metadataArg = ArgumentCaptor.forClass(ArtifactMetadata.class); + verify(metadataRepository, times(deletedVersions.size())).removeArtifact(metadataArg.capture(), eq(projectVersion)); + List metaL = metadataArg.getAllValues(); + for (ArtifactMetadata meta : metaL) { + assertTrue(meta.getId().startsWith(projectName)); + assertTrue(deletedVersions.contains(meta.getVersion())); + } + assertDeleted( versionRoot + "/maven-assembly-plugin-1.1.2-20070427.065136-1.jar" ); assertDeleted( versionRoot + "/maven-assembly-plugin-1.1.2-20070427.065136-1.jar.sha1" ); -- 2.39.5