From cb96e18f97711202e3c33a00aca2095a02b582a1 Mon Sep 17 00:00:00 2001 From: Brett Porter Date: Wed, 2 Dec 2009 02:21:05 +0000 Subject: [PATCH] [MRM-1025] change duplicate artifact reporting to be a repository scanned consumer to facilitate removing the database unprocessed consumers git-svn-id: https://svn.apache.org/repos/asf/archiva/branches/MRM-1025@886035 13f79535-47bb-0310-9956-ffa450edef68 --- .../ProjectModelToDatabaseConsumer.java | 5 +- .../archiva-artifact-reports/pom.xml | 4 + .../artifact/DuplicateArtifactsConsumer.java | 109 ++++++++++++------ .../artifact/DuplicateArtifactReportTest.java | 50 ++++++-- .../project/ProjectDependenciesConsumer.java | 100 ---------------- 5 files changed, 120 insertions(+), 148 deletions(-) delete mode 100644 archiva-modules/archiva-reporting/archiva-project-reports/src/main/java/org/apache/maven/archiva/reporting/project/ProjectDependenciesConsumer.java diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ProjectModelToDatabaseConsumer.java b/archiva-modules/archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ProjectModelToDatabaseConsumer.java index 3a8211a6d..4bfedb39a 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ProjectModelToDatabaseConsumer.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-database-consumers/src/main/java/org/apache/maven/archiva/consumers/database/ProjectModelToDatabaseConsumer.java @@ -9,7 +9,7 @@ package org.apache.maven.archiva.consumers.database; * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -34,10 +34,7 @@ import org.apache.maven.archiva.database.updater.DatabaseUnprocessedArtifactCons import org.apache.maven.archiva.model.ArchivaArtifact; import org.apache.maven.archiva.model.ArchivaModelCloner; import org.apache.maven.archiva.model.ArchivaProjectModel; -import org.apache.maven.archiva.model.CiManagement; -import org.apache.maven.archiva.model.IssueManagement; import org.apache.maven.archiva.model.Keys; -import org.apache.maven.archiva.model.Organization; import org.apache.maven.archiva.model.RepositoryProblem; import org.apache.maven.archiva.reporting.artifact.CorruptArtifactReport; import org.apache.maven.archiva.repository.ManagedRepositoryContent; diff --git a/archiva-modules/archiva-reporting/archiva-artifact-reports/pom.xml b/archiva-modules/archiva-reporting/archiva-artifact-reports/pom.xml index bc804e39c..ab6a66d52 100755 --- a/archiva-modules/archiva-reporting/archiva-artifact-reports/pom.xml +++ b/archiva-modules/archiva-reporting/archiva-artifact-reports/pom.xml @@ -40,6 +40,10 @@ org.apache.archiva archiva-repository-layer + + org.codehaus.plexus + plexus-digest + org.codehaus.plexus plexus-utils diff --git a/archiva-modules/archiva-reporting/archiva-artifact-reports/src/main/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactsConsumer.java b/archiva-modules/archiva-reporting/archiva-artifact-reports/src/main/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactsConsumer.java index 145285364..405b5d679 100644 --- a/archiva-modules/archiva-reporting/archiva-artifact-reports/src/main/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactsConsumer.java +++ b/archiva-modules/archiva-reporting/archiva-artifact-reports/src/main/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactsConsumer.java @@ -19,25 +19,32 @@ package org.apache.maven.archiva.reporting.artifact; * under the License. */ +import java.io.File; import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; import java.util.List; import org.apache.commons.collections.CollectionUtils; import org.apache.maven.archiva.configuration.ArchivaConfiguration; import org.apache.maven.archiva.configuration.ConfigurationNames; import org.apache.maven.archiva.configuration.FileTypes; +import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; import org.apache.maven.archiva.consumers.AbstractMonitoredConsumer; import org.apache.maven.archiva.consumers.ConsumerException; +import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer; import org.apache.maven.archiva.database.ArchivaDAO; import org.apache.maven.archiva.database.ArchivaDatabaseException; import org.apache.maven.archiva.database.ObjectNotFoundException; import org.apache.maven.archiva.database.constraints.ArtifactsByChecksumConstraint; -import org.apache.maven.archiva.database.updater.ArchivaArtifactConsumer; import org.apache.maven.archiva.model.ArchivaArtifact; import org.apache.maven.archiva.model.RepositoryProblem; import org.apache.maven.archiva.repository.ManagedRepositoryContent; import org.apache.maven.archiva.repository.RepositoryContentFactory; import org.apache.maven.archiva.repository.RepositoryException; +import org.apache.maven.archiva.repository.layout.LayoutException; +import org.codehaus.plexus.digest.Digester; +import org.codehaus.plexus.digest.DigesterException; import org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable; import org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException; import org.codehaus.plexus.registry.Registry; @@ -48,17 +55,19 @@ import org.slf4j.LoggerFactory; /** * Search the database of known SHA1 Checksums for potential duplicate artifacts. * + * TODO: no need for this to be a scanner - we can just query the database / content repository to get a full list + * * @version $Id$ - * - * @plexus.component role="org.apache.maven.archiva.database.updater.ArchivaArtifactConsumer" - * role-hint="duplicate-artifacts" + * @plexus.component role="org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer" + * role-hint="duplicate-artifacts" + * instantiation-strategy="per-lookup" */ public class DuplicateArtifactsConsumer extends AbstractMonitoredConsumer - implements ArchivaArtifactConsumer, RegistryListener, Initializable + implements KnownRepositoryContentConsumer, RegistryListener, Initializable { private Logger log = LoggerFactory.getLogger( DuplicateArtifactsConsumer.class ); - + /** * @plexus.configuration default-value="duplicate-artifacts" */ @@ -89,9 +98,19 @@ public class DuplicateArtifactsConsumer */ private RepositoryContentFactory repositoryFactory; - // TODO: why is this not used? If it should be, what about excludes? private List includes = new ArrayList(); + private File repositoryDir; + + /** + * @plexus.requirement role-hint="sha1" + */ + private Digester digestSha1; + + private String repoId; + + private ManagedRepositoryContent repository; + public String getId() { return id; @@ -107,40 +126,62 @@ public class DuplicateArtifactsConsumer return false; } - public void beginScan() + public List getIncludes() { - /* do nothing */ + return includes; } - public void completeScan() + public List getExcludes() { - /* do nothing */ + return Collections.emptyList(); } - public List getIncludedTypes() + public void beginScan( ManagedRepositoryConfiguration repo, Date whenGathered ) + throws ConsumerException { - return null; + try + { + repoId = repo.getId(); + repository = repositoryFactory.getManagedRepositoryContent( repoId ); + this.repositoryDir = new File( repository.getRepoRoot() ); + } + catch ( RepositoryException e ) + { + throw new ConsumerException( e.getMessage(), e ); + } } - public void processArchivaArtifact( ArchivaArtifact artifact ) + public void processFile( String path ) throws ConsumerException { - String checksumSha1 = artifact.getModel().getChecksumSHA1(); + File artifactFile = new File( this.repositoryDir, path ); - List results = null; + // TODO: would be quicker to somehow make sure it ran after the update database consumer, or as a part of that + // perhaps could use an artifact context that is retained for all consumers? First in can set the SHA-1 + String checksumSha1; try { - results = dao.getArtifactDAO().queryArtifacts( new ArtifactsByChecksumConstraint( - checksumSha1, ArtifactsByChecksumConstraint.SHA1 ) ); + checksumSha1 = digestSha1.calc( artifactFile ); + } + catch ( DigesterException e ) + { + throw new ConsumerException( e.getMessage(), e ); + } + + List results; + try + { + results = dao.getArtifactDAO().queryArtifacts( + new ArtifactsByChecksumConstraint( checksumSha1, ArtifactsByChecksumConstraint.SHA1 ) ); } catch ( ObjectNotFoundException e ) { - log.debug( "No duplicates for artifact: " + artifact ); + log.debug( "No duplicates for artifact: " + path + " (repository " + repoId + ")" ); return; } catch ( ArchivaDatabaseException e ) { - log.warn( "Unable to query DB for potential duplicates with : " + artifact ); + log.warn( "Unable to query DB for potential duplicates with: " + path + " (repository " + repoId + "): " + e.getMessage(), e ); return; } @@ -149,10 +190,20 @@ public class DuplicateArtifactsConsumer if ( results.size() <= 1 ) { // No duplicates detected. - log.debug( "Found no duplicate artifact results on: " + artifact ); + log.debug( "Found no duplicate artifact results on: " + path + " (repository " + repoId + ")" ); return; } + ArchivaArtifact artifact; + try + { + artifact = new ArchivaArtifact( repository.toArtifactReference( path ), repoId ); + } + catch ( LayoutException e ) + { + log.warn( "Unable to report problem for path: " + path ); + return; + } for ( ArchivaArtifact dupArtifact : results ) { if ( dupArtifact.equals( artifact ) ) @@ -163,7 +214,7 @@ public class DuplicateArtifactsConsumer RepositoryProblem problem = new RepositoryProblem(); problem.setRepositoryId( dupArtifact.getModel().getRepositoryId() ); - problem.setPath( toPath( dupArtifact ) ); + problem.setPath( path ); problem.setGroupId( artifact.getGroupId() ); problem.setArtifactId( artifact.getArtifactId() ); problem.setVersion( artifact.getVersion() ); @@ -186,19 +237,9 @@ public class DuplicateArtifactsConsumer } } - private String toPath( ArchivaArtifact artifact ) + public void completeScan() { - try - { - String repoId = artifact.getModel().getRepositoryId(); - ManagedRepositoryContent repo = repositoryFactory.getManagedRepositoryContent( repoId ); - return repo.toPath( artifact ); - } - catch ( RepositoryException e ) - { - log.warn( "Unable to calculate path for artifact: " + artifact ); - return ""; - } + // nothing to do } public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue ) diff --git a/archiva-modules/archiva-reporting/archiva-artifact-reports/src/test/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactReportTest.java b/archiva-modules/archiva-reporting/archiva-artifact-reports/src/test/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactReportTest.java index 4f44f7311..b905f8220 100644 --- a/archiva-modules/archiva-reporting/archiva-artifact-reports/src/test/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactReportTest.java +++ b/archiva-modules/archiva-reporting/archiva-artifact-reports/src/test/java/org/apache/maven/archiva/reporting/artifact/DuplicateArtifactReportTest.java @@ -20,17 +20,20 @@ package org.apache.maven.archiva.reporting.artifact; */ import java.io.File; +import java.io.IOException; import java.util.Date; import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.maven.archiva.configuration.ArchivaConfiguration; import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration; +import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer; import org.apache.maven.archiva.database.ArtifactDAO; -import org.apache.maven.archiva.database.updater.ArchivaArtifactConsumer; import org.apache.maven.archiva.model.ArchivaArtifact; import org.apache.maven.archiva.model.RepositoryProblem; import org.apache.maven.archiva.reporting.DynamicReportSource; +import org.apache.maven.archiva.repository.ManagedRepositoryContent; +import org.apache.maven.archiva.repository.RepositoryContentFactory; /** * DuplicateArtifactReportTest @@ -42,14 +45,18 @@ public class DuplicateArtifactReportTest { private static final String TESTABLE_REPO = "testable"; - private static final String HASH3 = "f3f653289f3217c65324830ab3415bc92feddefa"; + private static final String HASH3 = "94ca33031e37aa3f3b67e5b921c729f08a6bba75"; - private static final String HASH2 = "a49810ad3eba8651677ab57cd40a0f76fdef9538"; + private static final String HASH2 = "43f7aa390f1a0265fc2de7010133951c0718a67e"; - private static final String HASH1 = "232f01b24b1617c46a3d4b0ab3415bc9237dcdec"; + private static final String HASH1 = "8107759ababcbfa34bcb02bc4309caf6354982ab"; private ArtifactDAO artifactDao; + private ManagedRepositoryConfiguration repoConfig; + + private ManagedRepositoryContent content; + @Override protected void setUp() throws Exception @@ -60,19 +67,41 @@ public class DuplicateArtifactReportTest ArchivaConfiguration config = (ArchivaConfiguration) lookup( ArchivaConfiguration.class.getName(), "default" ); - ManagedRepositoryConfiguration repoConfig = new ManagedRepositoryConfiguration(); + repoConfig = new ManagedRepositoryConfiguration(); repoConfig.setId( TESTABLE_REPO ); repoConfig.setLayout( "default" ); File testRepoDir = new File( getBasedir(), "target/test-repository" ); FileUtils.forceMkdir( testRepoDir ); repoConfig.setLocation( testRepoDir.getAbsolutePath() ); config.getConfiguration().addManagedRepository( repoConfig ); + + RepositoryContentFactory factory = (RepositoryContentFactory) lookup( RepositoryContentFactory.class ); + content = factory.getManagedRepositoryContent( TESTABLE_REPO ); + + createArtifactFile( testRepoDir, "test-one", "1.0", "value1" ); + createArtifactFile( testRepoDir, "test-one", "1.1", "value1" ); + createArtifactFile( testRepoDir, "test-one", "1.2", "value1" ); + createArtifactFile( testRepoDir, "test-two", "1.0", "value1" ); + createArtifactFile( testRepoDir, "test-two", "2.0", "value3" ); + createArtifactFile( testRepoDir, "test-two", "2.1", "value2" ); + createArtifactFile( testRepoDir, "test-two", "3.0", "value2" ); + } + + private void createArtifactFile( File testRepoDir, String artifactId, String version, String value ) + throws IOException + { + File file = new File( testRepoDir, + "org/apache/maven/archiva/test/" + artifactId + "/" + version + "/" + artifactId + "-" + + version + ".jar" ); + file.getParentFile().mkdirs(); + FileUtils.writeStringToFile( file, value ); } public ArchivaArtifact createArtifact( String artifactId, String version ) { ArchivaArtifact artifact = - artifactDao.createArtifact( "org.apache.maven.archiva.test", artifactId, version, "", "jar", TESTABLE_REPO ); + artifactDao.createArtifact( "org.apache.maven.archiva.test", artifactId, version, "", "jar", + TESTABLE_REPO ); artifact.getModel().setLastModified( new Date() ); return artifact; } @@ -142,14 +171,15 @@ public class DuplicateArtifactReportTest throws Exception { List artifacts = dao.getArtifactDAO().queryArtifacts( null ); - ArchivaArtifactConsumer consumer = - (ArchivaArtifactConsumer) lookup( ArchivaArtifactConsumer.class.getName(), "duplicate-artifacts" ); - consumer.beginScan(); + KnownRepositoryContentConsumer consumer = + (KnownRepositoryContentConsumer) lookup( KnownRepositoryContentConsumer.class.getName(), + "duplicate-artifacts" ); + consumer.beginScan( repoConfig, new Date() ); try { for ( ArchivaArtifact artifact : artifacts ) { - consumer.processArchivaArtifact( artifact ); + consumer.processFile( content.toPath( artifact ) ); } } finally diff --git a/archiva-modules/archiva-reporting/archiva-project-reports/src/main/java/org/apache/maven/archiva/reporting/project/ProjectDependenciesConsumer.java b/archiva-modules/archiva-reporting/archiva-project-reports/src/main/java/org/apache/maven/archiva/reporting/project/ProjectDependenciesConsumer.java deleted file mode 100644 index 9a46dd1d1..000000000 --- a/archiva-modules/archiva-reporting/archiva-project-reports/src/main/java/org/apache/maven/archiva/reporting/project/ProjectDependenciesConsumer.java +++ /dev/null @@ -1,100 +0,0 @@ -package org.apache.maven.archiva.reporting.project; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.apache.maven.archiva.consumers.AbstractMonitoredConsumer; -import org.apache.maven.archiva.consumers.ConsumerException; -import org.apache.maven.archiva.database.updater.ArchivaArtifactConsumer; -import org.apache.maven.archiva.model.ArchivaArtifact; - -import java.util.ArrayList; -import java.util.List; - -/** - * ProjectDependenciesConsumer - * - * @version $Id$ - * - * @plexus.component role="org.apache.maven.archiva.database.updater.ArchivaArtifactConsumer" - * role-hint="missing-dependencies" - */ -public class ProjectDependenciesConsumer - extends AbstractMonitoredConsumer - implements ArchivaArtifactConsumer -{ - /** - * @plexus.configuration default-value="missing-dependencies" - */ - private String id; - - /** - * @plexus.configuration default-value="Check for missing dependencies." - */ - private String description; - - private List includes; - - public ProjectDependenciesConsumer() - { - this.includes = new ArrayList(); - this.includes.add( "pom" ); - } - - public String getId() - { - return id; - } - - public String getDescription() - { - return description; - } - - public boolean isPermanent() - { - return false; - } - - public void beginScan() - { - /* do nothing */ - } - - public void completeScan() - { - /* do nothing */ - } - - public List getIncludedTypes() - { - return includes; - } - - public void processArchivaArtifact( ArchivaArtifact artifact ) - throws ConsumerException - { - // TODO: consider loading this logic into the 'update-db-project' consumer. - - // TODO: Load the ArchivaProjectModel. - // TODO: Attach a monitor for missing parent poms to resolvers / filters. - // TODO: Attach a monitor for missing dependencies to resolvers / filters. - // TODO: Fully resolve the ArchivaProjectModel and listen on monitors. - } -} -- 2.39.5