From 21b1ac804be223d0702bd9dc774c14214d61f0c3 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Wed, 9 May 2007 01:47:52 +0000 Subject: [PATCH] Updates to RepositoryScanning process, task, and admin screen git-svn-id: https://svn.apache.org/repos/asf/maven/archiva/trunk@536390 13f79535-47bb-0310-9956-ffa450edef68 --- .../archiva/configuration/FileTypes.java | 11 +- .../ArchivaConfigurationTest.java | 6 +- archiva-base/archiva-converter/pom.xml | 4 + .../DefaultLegacyRepositoryConverter.java | 21 +- .../scanner/DefaultRepositoryScanner.java | 128 ++++++++++ .../RepositoryContentConsumerUtil.java | 25 +- .../repository/scanner/RepositoryScanner.java | 178 ++++---------- .../scanner/RepositoryScannerInstance.java | 157 +++++------- .../functors/ConsumerProcessFileClosure.java | 89 +++++++ .../functors/ConsumerWantsFilePredicate.java | 122 ++++++++++ .../functors/TriggerBeginScanClosure.java | 63 +++++ .../scanner/InvalidScanConsumer.java | 92 ++++++++ ...anConsumer.java => KnownScanConsumer.java} | 11 +- .../scanner/RepositoryScannerTest.java | 124 ++++++---- .../apache/maven/archiva/cli/ArchivaCli.java | 37 +-- .../ArchivaScheduledTaskExecutor.java | 59 ++--- .../scanning/RepositoryScanningAction.java | 63 ++++- .../WEB-INF/jsp/admin/repositoryScanning.jsp | 223 +++++++++++------- .../src/main/webapp/css/site.css | 20 +- 19 files changed, 986 insertions(+), 447 deletions(-) create mode 100644 archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/DefaultRepositoryScanner.java create mode 100644 archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/ConsumerProcessFileClosure.java create mode 100644 archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/ConsumerWantsFilePredicate.java create mode 100644 archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/TriggerBeginScanClosure.java create mode 100644 archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/InvalidScanConsumer.java rename archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/{ScanConsumer.java => KnownScanConsumer.java} (87%) diff --git a/archiva-base/archiva-configuration/src/main/java/org/apache/maven/archiva/configuration/FileTypes.java b/archiva-base/archiva-configuration/src/main/java/org/apache/maven/archiva/configuration/FileTypes.java index 783b35885..262b61468 100644 --- a/archiva-base/archiva-configuration/src/main/java/org/apache/maven/archiva/configuration/FileTypes.java +++ b/archiva-base/archiva-configuration/src/main/java/org/apache/maven/archiva/configuration/FileTypes.java @@ -49,11 +49,13 @@ public class FileTypes implements Initializable { public static final String ARTIFACTS = "artifacts"; - + public static final String AUTO_REMOVE = "auto-remove"; - + public static final String INDEXABLE_CONTENT = "indexable-content"; - + + public static final String IGNORED = "ignored"; + /** * @plexus.requirement */ @@ -110,7 +112,8 @@ public class FileTypes try { - URL defaultArchivaXml = this.getClass().getResource( "" ); + URL defaultArchivaXml = this.getClass() + .getResource( "/org/apache/maven/archiva/configuration/default-archiva.xml" ); XMLReader reader = new XMLReader( "configuration", defaultArchivaXml ); List resp = reader.getElementList( "//configuration/repositoryScanning/fileTypes/fileType" ); diff --git a/archiva-base/archiva-configuration/src/test/java/org/apache/maven/archiva/configuration/ArchivaConfigurationTest.java b/archiva-base/archiva-configuration/src/test/java/org/apache/maven/archiva/configuration/ArchivaConfigurationTest.java index 0ce132e36..180bc1f8f 100644 --- a/archiva-base/archiva-configuration/src/test/java/org/apache/maven/archiva/configuration/ArchivaConfigurationTest.java +++ b/archiva-base/archiva-configuration/src/test/java/org/apache/maven/archiva/configuration/ArchivaConfigurationTest.java @@ -32,15 +32,11 @@ import java.util.List; */ public class ArchivaConfigurationTest extends PlexusTestCase { - private FileTypes filetypes; - public void testDefaults() throws Exception { ArchivaConfiguration archivaConfiguration = (ArchivaConfiguration) lookup( ArchivaConfiguration.class, "test-defaults" ); - filetypes = (FileTypes) lookup( FileTypes.class ); - Configuration configuration = archivaConfiguration.getConfiguration(); // check default configuration @@ -53,6 +49,8 @@ public class ArchivaConfigurationTest extends PlexusTestCase ArchivaConfiguration archivaConfiguration = (ArchivaConfiguration) lookup( ArchivaConfiguration.class.getName(), "test-configuration" ); + FileTypes filetypes = (FileTypes) lookup( FileTypes.class.getName() ); + Configuration configuration = archivaConfiguration.getConfiguration(); assertEquals( "check repositories", 4, configuration.getRepositories().size() ); diff --git a/archiva-base/archiva-converter/pom.xml b/archiva-base/archiva-converter/pom.xml index dbfeca6cd..6ae0fe21a 100644 --- a/archiva-base/archiva-converter/pom.xml +++ b/archiva-base/archiva-converter/pom.xml @@ -71,6 +71,10 @@ commons-io commons-io + + org.codehaus.plexus.registry + plexus-registry-commons + hsqldb diff --git a/archiva-base/archiva-converter/src/main/java/org/apache/maven/archiva/converter/legacy/DefaultLegacyRepositoryConverter.java b/archiva-base/archiva-converter/src/main/java/org/apache/maven/archiva/converter/legacy/DefaultLegacyRepositoryConverter.java index 4926f5aa8..bdabf5a1b 100644 --- a/archiva-base/archiva-converter/src/main/java/org/apache/maven/archiva/converter/legacy/DefaultLegacyRepositoryConverter.java +++ b/archiva-base/archiva-converter/src/main/java/org/apache/maven/archiva/converter/legacy/DefaultLegacyRepositoryConverter.java @@ -31,6 +31,8 @@ import org.apache.maven.artifact.repository.layout.ArtifactRepositoryLayout; import java.io.File; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; /** @@ -54,11 +56,16 @@ public class DefaultLegacyRepositoryConverter private ArtifactRepositoryLayout defaultLayout; /** - * @plexus.requirement role="org.apache.maven.archiva.consumers.RepositoryContentConsumer" + * @plexus.requirement role="org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer" * role-hint="artifact-legacy-to-default-converter" */ private LegacyConverterArtifactConsumer legacyConverterConsumer; + /** + * @plexus.requirement + */ + private RepositoryScanner repoScanner; + public void convertLegacyRepository( File legacyRepositoryDirectory, File repositoryDirectory, List fileExclusionPatterns ) throws RepositoryConversionException @@ -90,11 +97,15 @@ public class DefaultLegacyRepositoryConverter legacyConverterConsumer.setExcludes( fileExclusionPatterns ); legacyConverterConsumer.setDestinationRepository( repository ); - List consumers = new ArrayList(); - consumers.add( legacyConverterConsumer ); + List knownConsumers = new ArrayList(); + knownConsumers.add( legacyConverterConsumer ); + + List invalidConsumers = Collections.EMPTY_LIST; + List ignoredContent = new ArrayList(); + ignoredContent.addAll( Arrays.asList( RepositoryScanner.IGNORABLE_CONTENT ) ); - RepositoryScanner scanner = new RepositoryScanner(); - scanner.scan( legacyRepository, consumers, true ); + repoScanner.scan( legacyRepository, knownConsumers, invalidConsumers, ignoredContent, + RepositoryScanner.FRESH_SCAN ); } catch ( RepositoryException e ) { diff --git a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/DefaultRepositoryScanner.java b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/DefaultRepositoryScanner.java new file mode 100644 index 000000000..45dab28e0 --- /dev/null +++ b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/DefaultRepositoryScanner.java @@ -0,0 +1,128 @@ +package org.apache.maven.archiva.repository.scanner; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.commons.collections.CollectionUtils; +import org.apache.maven.archiva.configuration.FileTypes; +import org.apache.maven.archiva.model.ArchivaRepository; +import org.apache.maven.archiva.model.RepositoryContentStatistics; +import org.apache.maven.archiva.repository.RepositoryException; +import org.codehaus.plexus.logging.AbstractLogEnabled; +import org.codehaus.plexus.util.DirectoryWalker; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +/** + * DefaultRepositoryScanner + * + * @author Joakim Erdfelt + * @version $Id$ + * + * @plexus.component role="org.apache.maven.archiva.repository.scanner.RepositoryScanner" + */ +public class DefaultRepositoryScanner + extends AbstractLogEnabled + implements RepositoryScanner +{ + /** + * @plexus.requirement + */ + private FileTypes filetypes; + + /** + * @plexus.requirement + */ + private RepositoryContentConsumerUtil consumerUtil; + + public RepositoryContentStatistics scan( ArchivaRepository repository, long changesSince ) + throws RepositoryException + { + List knownContentConsumers = consumerUtil.getSelectedKnownConsumers(); + List invalidContentConsumers = consumerUtil.getSelectedInvalidConsumers(); + List ignoredPatterns = filetypes.getFileTypePatterns( FileTypes.IGNORED ); + + return scan( repository, knownContentConsumers, invalidContentConsumers, ignoredPatterns, changesSince ); + } + + public RepositoryContentStatistics scan( ArchivaRepository repository, List knownContentConsumers, + List invalidContentConsumers, List ignoredContentPatterns, + long changesSince ) + throws RepositoryException + { + if ( repository == null ) + { + throw new IllegalArgumentException( "Unable to operate on a null repository." ); + } + + if ( !repository.isManaged() ) + { + throw new UnsupportedOperationException( "Only filesystem repositories are supported." ); + } + + File repositoryBase = new File( repository.getUrl().getPath() ); + + if ( !repositoryBase.exists() ) + { + throw new UnsupportedOperationException( "Unable to scan a repository, directory " + + repositoryBase.getAbsolutePath() + " does not exist." ); + } + + if ( !repositoryBase.isDirectory() ) + { + throw new UnsupportedOperationException( "Unable to scan a repository, path " + + repositoryBase.getAbsolutePath() + " is not a directory." ); + } + + // Setup Includes / Excludes. + + List allExcludes = new ArrayList(); + List allIncludes = new ArrayList(); + + if ( CollectionUtils.isNotEmpty( ignoredContentPatterns ) ) + { + allExcludes.addAll( ignoredContentPatterns ); + } + + // Scan All Content. (intentional) + allIncludes.add( "**/*" ); + + // Setup Directory Walker + DirectoryWalker dirWalker = new DirectoryWalker(); + + dirWalker.setBaseDir( repositoryBase ); + + dirWalker.setIncludes( allIncludes ); + dirWalker.setExcludes( allExcludes ); + + // Setup the Scan Instance + RepositoryScannerInstance scannerInstance = new RepositoryScannerInstance( repository, knownContentConsumers, + invalidContentConsumers, getLogger() ); + scannerInstance.setOnlyModifiedAfterTimestamp( changesSince ); + + dirWalker.addDirectoryWalkListener( scannerInstance ); + + // Execute scan. + dirWalker.scan(); + + return scannerInstance.getStatistics(); + } +} diff --git a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryContentConsumerUtil.java b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryContentConsumerUtil.java index ec03a3437..135db420d 100644 --- a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryContentConsumerUtil.java +++ b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryContentConsumerUtil.java @@ -30,6 +30,7 @@ import org.apache.maven.archiva.consumers.InvalidRepositoryContentConsumer; import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer; import org.apache.maven.archiva.consumers.RepositoryContentConsumer; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -128,7 +129,7 @@ public class RepositoryContentConsumerUtil return new SelectedInvalidRepoConsumersPredicate(); } - public Map getSelectedKnownConsumers() + public Map getSelectedKnownConsumersMap() { RepositoryScanningConfiguration scanning = archivaConfiguration.getConfiguration().getRepositoryScanning(); @@ -139,7 +140,7 @@ public class RepositoryContentConsumerUtil return consumerMapClosure.getMap(); } - public Map getSelectedInvalidConsumers() + public Map getSelectedInvalidConsumersMap() { RepositoryScanningConfiguration scanning = archivaConfiguration.getConfiguration().getRepositoryScanning(); @@ -149,6 +150,26 @@ public class RepositoryContentConsumerUtil return consumerMapClosure.getMap(); } + + public List getSelectedKnownConsumers() + { + RepositoryScanningConfiguration scanning = archivaConfiguration.getConfiguration().getRepositoryScanning(); + + List ret = new ArrayList(); + ret.addAll( CollectionUtils.select( scanning.getGoodConsumers(), getKnownSelectionPredicate() )); + + return ret; + } + + public List getSelectedInvalidConsumers() + { + RepositoryScanningConfiguration scanning = archivaConfiguration.getConfiguration().getRepositoryScanning(); + + List ret = new ArrayList(); + ret.addAll( CollectionUtils.select( scanning.getBadConsumers(), getInvalidSelectionPredicate() )); + + return ret; + } public List getAvailableKnownConsumers() { diff --git a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScanner.java b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScanner.java index e2ed6768e..12f00244a 100644 --- a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScanner.java +++ b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScanner.java @@ -19,17 +19,12 @@ package org.apache.maven.archiva.repository.scanner; * under the License. */ -import org.apache.maven.archiva.consumers.RepositoryContentConsumer; +import org.apache.maven.archiva.consumers.InvalidRepositoryContentConsumer; +import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer; import org.apache.maven.archiva.model.ArchivaRepository; import org.apache.maven.archiva.model.RepositoryContentStatistics; import org.apache.maven.archiva.repository.RepositoryException; -import org.codehaus.plexus.util.DirectoryWalker; -import org.codehaus.plexus.util.FileUtils; -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; import java.util.List; /** @@ -38,17 +33,34 @@ import java.util.List; * @author Joakim Erdfelt * @version $Id$ */ -public class RepositoryScanner +public interface RepositoryScanner { /** - * Standard patterns to exclude from discovery as they are usually noise. + * The value to pass to {@link #scan(ArchivaRepository, long)} to have the scan + * operate in a fresh fashion, with no check on changes based on timestamp. */ - private static final String[] STANDARD_SCANNER_EXCLUDES = { + public static final long FRESH_SCAN = 0; + + /** + *

+ * Typical Ignorable Content patterns. + *

+ * + *

+ * NOTE: Do not use for normal webapp or task driven repository scanning. + *

+ * + *

+ * These patterns are only valid for archiva-cli and archiva-converter use. + *

+ */ + public static final String[] IGNORABLE_CONTENT = { "bin/**", "reports/**", ".index", ".reports/**", ".maven/**", + "**/.svn/**", "**/*snapshot-version", "*/website/**", "*/licences/**", @@ -58,140 +70,42 @@ public class RepositoryScanner "**/README*", "**/CHANGELOG*", "**/KEYS*" }; - + /** - * Walk the repository, and report to the consumers the files found. - * - * Report changes to the appropriate Consumer. + * Scan the repository for content changes. * - * This is just a convenience method to {@link #scan(ArtifactRepository, List, boolean, long, List, List)} - * equivalent to calling scan( repository, consumers, includeSnapshots, 0, null, null ); + * Internally, this will use the as-configured known and invalid consumer lists. * * @param repository the repository to change. - * @param consumers use the provided list of consumers. - * @param includeSnapshots true to include snapshots in the walking of this repository. + * @param changesSince the timestamp to use as a threshold on what is considered new or changed. + * (To have all content be taken into consideration regardless of timestamp, + * use the {@link #FRESH_SCAN} constant) * @return the statistics for this scan. * @throws RepositoryException if there was a fundamental problem with getting the discoverer started. */ - public RepositoryContentStatistics scan( ArchivaRepository repository, List consumers, boolean includeSnapshots ) - throws RepositoryException - { - return scan( repository, consumers, includeSnapshots, 0, null, null ); - } - + public RepositoryContentStatistics scan( ArchivaRepository repository, long changesSince ) + throws RepositoryException; + /** - * Walk the repository, and report to the consumers the files found. + * Scan the repository for content changes. * - * Report changes to the appropriate Consumer. + * Internally, this will use the as-configured known and invalid consumer lists. * * @param repository the repository to change. - * @param consumers use the provided list of consumers. - * @param includeSnapshots true to include snapshots in the scanning of this repository. - * @param onlyModifiedAfterTimestamp Only report to the consumers, files that have a {@link File#lastModified()}) - * after the provided timestamp. - * @param extraFileExclusions an optional list of file exclusions on the walk. - * @param extraFileInclusions an optional list of file inclusions on the walk. + * @param knownContentConsumers the list of consumers that follow the {@link KnownRepositoryContentConsumer} + * interface that should be used for this scan. + * @param invalidContentConsumers the list of consumers that follow the {@link InvalidRepositoryContentConsumer} + * interface that should be used for this scan. + * @param ignoredContentPatterns list of patterns that should be ignored and not sent to any consumer. + * @param changesSince the timestamp to use as a threshold on what is considered new or changed. + * (To have all content be taken into consideration regardless of timestamp, + * use the {@link #FRESH_SCAN} constant) * @return the statistics for this scan. - * @throws RepositoryException if there was a fundamental problem with getting the discoverer started. + * @throws RepositoryException if there was a fundamental problem with getting the discoverer started. */ - public RepositoryContentStatistics scan( ArchivaRepository repository, List consumers, boolean includeSnapshots, - long onlyModifiedAfterTimestamp, List extraFileExclusions, List extraFileInclusions ) - throws RepositoryException - { - if ( repository == null ) - { - throw new IllegalArgumentException( "Unable to operate on a null repository." ); - } - - if ( !"file".equals( repository.getUrl().getProtocol() ) ) - { - throw new UnsupportedOperationException( "Only filesystem repositories are supported." ); - } - - File repositoryBase = new File( repository.getUrl().getPath() ); - - if ( !repositoryBase.exists() ) - { - throw new UnsupportedOperationException( "Unable to scan a repository, directory " - + repositoryBase.getAbsolutePath() + " does not exist." ); - } - - if ( !repositoryBase.isDirectory() ) - { - throw new UnsupportedOperationException( "Unable to scan a repository, path " - + repositoryBase.getAbsolutePath() + " is not a directory." ); - } - - // Setup Includes / Excludes. - - List allExcludes = new ArrayList(); - List allIncludes = new ArrayList(); - - // Exclude all of the SCM patterns. - allExcludes.addAll( FileUtils.getDefaultExcludesAsList() ); - - // Exclude all of the archiva noise patterns. - allExcludes.addAll( Arrays.asList( STANDARD_SCANNER_EXCLUDES ) ); - - if ( !includeSnapshots ) - { - allExcludes.add( "**/*-SNAPSHOT*" ); - } - - if ( extraFileExclusions != null ) - { - allExcludes.addAll( extraFileExclusions ); - } - - Iterator it = consumers.iterator(); - while ( it.hasNext() ) - { - RepositoryContentConsumer consumer = (RepositoryContentConsumer) it.next(); - - /* NOTE: Do not insert the consumer exclusion patterns here. - * Exclusion patterns are handled by RepositoryScanner.wantsFile(Consumer, String) - * - * addUniqueElements( consumer.getExcludePatterns(), allExcludes ); - */ - addUniqueElements( consumer.getIncludes(), allIncludes ); - } - - if ( extraFileInclusions != null ) - { - allIncludes.addAll( extraFileInclusions ); - } - - // Setup Directory Walker - - DirectoryWalker dirWalker = new DirectoryWalker(); - - dirWalker.setBaseDir( repositoryBase ); - - dirWalker.setIncludes( allIncludes ); - dirWalker.setExcludes( allExcludes ); - - // Setup the Scan Instance - RepositoryScannerInstance scannerInstance = new RepositoryScannerInstance( repository, consumers ); - scannerInstance.setOnlyModifiedAfterTimestamp( onlyModifiedAfterTimestamp ); - - dirWalker.addDirectoryWalkListener( scannerInstance ); - - // Execute scan. - dirWalker.scan(); - - return scannerInstance.getStatistics(); - } + public RepositoryContentStatistics scan( ArchivaRepository repository, List knownContentConsumers, + List invalidContentConsumers, List ignoredContentPatterns, + long changesSince ) + throws RepositoryException; - private void addUniqueElements( List fromList, List toList ) - { - Iterator itFrom = fromList.iterator(); - while ( itFrom.hasNext() ) - { - Object o = itFrom.next(); - if ( !toList.contains( o ) ) - { - toList.add( o ); - } - } - } } diff --git a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerInstance.java b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerInstance.java index c5e498b8e..2c0637cb4 100644 --- a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerInstance.java +++ b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerInstance.java @@ -19,20 +19,20 @@ package org.apache.maven.archiva.repository.scanner; * under the License. */ +import org.apache.commons.collections.Closure; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.functors.IfClosure; import org.apache.commons.lang.SystemUtils; import org.apache.maven.archiva.common.utils.BaseFile; -import org.apache.maven.archiva.consumers.ConsumerException; -import org.apache.maven.archiva.consumers.RepositoryContentConsumer; import org.apache.maven.archiva.model.ArchivaRepository; import org.apache.maven.archiva.model.RepositoryContentStatistics; +import org.apache.maven.archiva.repository.scanner.functors.ConsumerProcessFileClosure; +import org.apache.maven.archiva.repository.scanner.functors.ConsumerWantsFilePredicate; +import org.apache.maven.archiva.repository.scanner.functors.TriggerBeginScanClosure; +import org.codehaus.plexus.logging.Logger; import org.codehaus.plexus.util.DirectoryWalkListener; -import org.codehaus.plexus.util.SelectorUtils; -import org.codehaus.plexus.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; -import java.util.Iterator; import java.util.List; /** @@ -41,45 +41,53 @@ import java.util.List; * @author Joakim Erdfelt * @version $Id$ */ -public class RepositoryScannerInstance implements DirectoryWalkListener +public class RepositoryScannerInstance + implements DirectoryWalkListener { - private static Logger log = LoggerFactory.getLogger( RepositoryScannerInstance.class ); - - private List consumers; + /** + * Consumers that process known content. + */ + private List knownConsumers; - private ArchivaRepository repository; + /** + * Consumers that process unknown/invalid content. + */ + private List invalidConsumers; - private boolean isCaseSensitive = true; + ArchivaRepository repository; private RepositoryContentStatistics stats; private long onlyModifiedAfterTimestamp = 0; - public RepositoryScannerInstance( ArchivaRepository repository, List consumerList ) + private ConsumerProcessFileClosure consumerProcessFile; + + private ConsumerWantsFilePredicate consumerWantsFile; + + private Logger logger; + + public RepositoryScannerInstance( ArchivaRepository repository, List knownConsumerList, List invalidConsumerList, + Logger logger ) { this.repository = repository; - this.consumers = consumerList; + this.knownConsumers = knownConsumerList; + this.invalidConsumers = invalidConsumerList; + this.logger = logger; + + this.consumerProcessFile = new ConsumerProcessFileClosure( logger ); + this.consumerWantsFile = new ConsumerWantsFilePredicate(); + stats = new RepositoryContentStatistics(); stats.setRepositoryId( repository.getId() ); - Iterator it = this.consumers.iterator(); - while ( it.hasNext() ) - { - RepositoryContentConsumer consumer = (RepositoryContentConsumer) it.next(); - try - { - consumer.beginScan( repository ); - } - catch ( ConsumerException e ) - { - // TODO: remove bad consumers from list - log.warn( "Consumer [" + consumer.getId() + "] cannot begin: " + e.getMessage(), e ); - } - } + Closure triggerBeginScan = new TriggerBeginScanClosure( repository, logger ); + + CollectionUtils.forAllDo( knownConsumerList, triggerBeginScan ); + CollectionUtils.forAllDo( invalidConsumerList, triggerBeginScan ); if ( SystemUtils.IS_OS_WINDOWS ) { - isCaseSensitive = false; + consumerWantsFile.setCaseSensitive( false ); } } @@ -90,13 +98,13 @@ public class RepositoryScannerInstance implements DirectoryWalkListener public void directoryWalkStarting( File basedir ) { - log.info( "Walk Started: [" + this.repository.getId() + "] " + this.repository.getUrl() ); + logger.info( "Walk Started: [" + this.repository.getId() + "] " + this.repository.getUrl() ); stats.triggerStart(); } public void directoryWalkStep( int percentage, File file ) { - log.debug( "Walk Step: " + percentage + ", " + file ); + logger.debug( "Walk Step: " + percentage + ", " + file ); stats.increaseFileCount(); @@ -104,86 +112,33 @@ public class RepositoryScannerInstance implements DirectoryWalkListener if ( file.lastModified() < onlyModifiedAfterTimestamp ) { // Skip file as no change has occured. - log.debug( "Skipping, No Change: " + file.getAbsolutePath() ); + logger.debug( "Skipping, No Change: " + file.getAbsolutePath() ); return; } - synchronized ( consumers ) + stats.increaseNewFileCount(); + + BaseFile basefile = new BaseFile( repository.getUrl().getPath(), file ); + + consumerProcessFile.setBasefile( basefile ); + consumerWantsFile.setBasefile( basefile ); + + Closure processIfWanted = IfClosure.getInstance( consumerWantsFile, consumerProcessFile ); + CollectionUtils.forAllDo( this.knownConsumers, processIfWanted ); + + if ( consumerWantsFile.getWantedFileCount() <= 0 ) { - stats.increaseNewFileCount(); - - BaseFile basefile = new BaseFile( repository.getUrl().getPath(), file ); - - Iterator itConsumers = this.consumers.iterator(); - while ( itConsumers.hasNext() ) - { - RepositoryContentConsumer consumer = (RepositoryContentConsumer) itConsumers.next(); - - if ( wantsFile( consumer, StringUtils.replace( basefile.getRelativePath(), "\\", "/" ) ) ) - { - try - { - log.debug( "Sending to consumer: " + consumer.getId() ); - consumer.processFile( basefile.getRelativePath() ); - } - catch ( Exception e ) - { - /* Intentionally Catch all exceptions. - * So that the discoverer processing can continue. - */ - log.error( "Consumer [" + consumer.getId() + "] had an error when processing file [" - + basefile.getAbsolutePath() + "]: " + e.getMessage(), e ); - } - } - else - { - log.debug( "Skipping consumer " + consumer.getId() + " for file " + basefile.getRelativePath() ); - } - } + // Nothing known processed this file. It is invalid! + CollectionUtils.forAllDo( this.invalidConsumers, consumerProcessFile ); } } public void directoryWalkFinished() { - log.info( "Walk Finished: [" + this.repository.getId() + "] " + this.repository.getUrl() ); + logger.info( "Walk Finished: [" + this.repository.getId() + "] " + this.repository.getUrl() ); stats.triggerFinished(); } - private boolean wantsFile( RepositoryContentConsumer consumer, String relativePath ) - { - Iterator it; - - // Test excludes first. - if ( consumer.getExcludes() != null ) - { - it = consumer.getExcludes().iterator(); - while ( it.hasNext() ) - { - String pattern = (String) it.next(); - if ( SelectorUtils.matchPath( pattern, relativePath, isCaseSensitive ) ) - { - // Definately does NOT WANT FILE. - return false; - } - } - } - - // Now test includes. - it = consumer.getIncludes().iterator(); - while ( it.hasNext() ) - { - String pattern = (String) it.next(); - if ( SelectorUtils.matchPath( pattern, relativePath, isCaseSensitive ) ) - { - // Specifically WANTS FILE. - return true; - } - } - - // Not included, and Not excluded? Default to EXCLUDE. - return false; - } - public long getOnlyModifiedAfterTimestamp() { return onlyModifiedAfterTimestamp; @@ -199,6 +154,6 @@ public class RepositoryScannerInstance implements DirectoryWalkListener */ public void debug( String message ) { - log.debug( "Repository Scanner: " + message ); + logger.debug( "Repository Scanner: " + message ); } } diff --git a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/ConsumerProcessFileClosure.java b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/ConsumerProcessFileClosure.java new file mode 100644 index 000000000..d79be78cf --- /dev/null +++ b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/ConsumerProcessFileClosure.java @@ -0,0 +1,89 @@ +package org.apache.maven.archiva.repository.scanner.functors; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.commons.collections.Closure; +import org.apache.maven.archiva.common.utils.BaseFile; +import org.apache.maven.archiva.consumers.RepositoryContentConsumer; +import org.codehaus.plexus.logging.Logger; + +/** + * ConsumerProcessFileClosure + * + * @author Joakim Erdfelt + * @version $Id$ + */ +public class ConsumerProcessFileClosure + implements Closure +{ + private BaseFile basefile; + + private Logger logger; + + public ConsumerProcessFileClosure( Logger logger ) + { + // Lame. I know, but seeing as plexus doesn't like to cleanup after + // application loaded/lookup'd components, this is the best I can do. + this.logger = logger; + } + + public void execute( Object input ) + { + if ( input instanceof RepositoryContentConsumer ) + { + RepositoryContentConsumer consumer = (RepositoryContentConsumer) input; + + try + { + logger.debug( "Sending to consumer: " + consumer.getId() ); + + consumer.processFile( basefile.getRelativePath() ); + } + catch ( Exception e ) + { + /* Intentionally Catch all exceptions. + * So that the discoverer processing can continue. + */ + logger.error( "Consumer [" + consumer.getId() + "] had an error when processing file [" + + basefile.getAbsolutePath() + "]: " + e.getMessage(), e ); + } + } + } + + public BaseFile getBasefile() + { + return basefile; + } + + public void setBasefile( BaseFile basefile ) + { + this.basefile = basefile; + } + + public Logger getLogger() + { + return logger; + } + + public void setLogger( Logger logger ) + { + this.logger = logger; + } +} diff --git a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/ConsumerWantsFilePredicate.java b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/ConsumerWantsFilePredicate.java new file mode 100644 index 000000000..0d2bc5719 --- /dev/null +++ b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/ConsumerWantsFilePredicate.java @@ -0,0 +1,122 @@ +package org.apache.maven.archiva.repository.scanner.functors; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.commons.collections.Predicate; +import org.apache.maven.archiva.common.utils.BaseFile; +import org.apache.maven.archiva.consumers.RepositoryContentConsumer; +import org.codehaus.plexus.util.SelectorUtils; +import org.codehaus.plexus.util.StringUtils; + +import java.util.Iterator; + +/** + * ConsumerWantsFilePredicate + * + * @author Joakim Erdfelt + * @version $Id$ + */ +public class ConsumerWantsFilePredicate + implements Predicate +{ + private BaseFile basefile; + + private boolean isCaseSensitive = true; + + private int wantedFileCount = 0; + + public boolean evaluate( Object object ) + { + boolean satisfies = false; + + if ( object instanceof RepositoryContentConsumer ) + { + RepositoryContentConsumer consumer = (RepositoryContentConsumer) object; + if ( wantsFile( consumer, StringUtils.replace( basefile.getRelativePath(), "\\", "/" ) ) ) + { + satisfies = true; + wantedFileCount++; + } + } + + return satisfies; + } + + public BaseFile getBasefile() + { + return basefile; + } + + public int getWantedFileCount() + { + return wantedFileCount; + } + + public boolean isCaseSensitive() + { + return isCaseSensitive; + } + + public void setBasefile( BaseFile basefile ) + { + this.basefile = basefile; + this.wantedFileCount = 0; + } + + public void setCaseSensitive( boolean isCaseSensitive ) + { + this.isCaseSensitive = isCaseSensitive; + } + + private boolean wantsFile( RepositoryContentConsumer consumer, String relativePath ) + { + Iterator it; + + // Test excludes first. + if ( consumer.getExcludes() != null ) + { + it = consumer.getExcludes().iterator(); + while ( it.hasNext() ) + { + String pattern = (String) it.next(); + if ( SelectorUtils.matchPath( pattern, relativePath, isCaseSensitive ) ) + { + // Definately does NOT WANT FILE. + return false; + } + } + } + + // Now test includes. + it = consumer.getIncludes().iterator(); + while ( it.hasNext() ) + { + String pattern = (String) it.next(); + if ( SelectorUtils.matchPath( pattern, relativePath, isCaseSensitive ) ) + { + // Specifically WANTS FILE. + return true; + } + } + + // Not included, and Not excluded? Default to EXCLUDE. + return false; + } +} diff --git a/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/TriggerBeginScanClosure.java b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/TriggerBeginScanClosure.java new file mode 100644 index 000000000..1b546bdf4 --- /dev/null +++ b/archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/TriggerBeginScanClosure.java @@ -0,0 +1,63 @@ +package org.apache.maven.archiva.repository.scanner.functors; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.commons.collections.Closure; +import org.apache.maven.archiva.consumers.ConsumerException; +import org.apache.maven.archiva.consumers.RepositoryContentConsumer; +import org.apache.maven.archiva.model.ArchivaRepository; +import org.codehaus.plexus.logging.Logger; + +/** + * TriggerBeginScanClosure + * + * @author Joakim Erdfelt + * @version $Id$ + */ +public class TriggerBeginScanClosure + implements Closure +{ + private ArchivaRepository repository; + + private Logger logger; + + public TriggerBeginScanClosure( ArchivaRepository repository, Logger logger ) + { + this.repository = repository; + this.logger = logger; + } + + public void execute( Object input ) + { + if ( input instanceof RepositoryContentConsumer ) + { + RepositoryContentConsumer consumer = (RepositoryContentConsumer) input; + + try + { + consumer.beginScan( repository ); + } + catch ( ConsumerException e ) + { + logger.warn( "Consumer [" + consumer.getId() + "] cannot begin: " + e.getMessage(), e ); + } + } + } +} \ No newline at end of file diff --git a/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/InvalidScanConsumer.java b/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/InvalidScanConsumer.java new file mode 100644 index 000000000..68bc9db95 --- /dev/null +++ b/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/InvalidScanConsumer.java @@ -0,0 +1,92 @@ +package org.apache.maven.archiva.repository.scanner; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.archiva.consumers.AbstractMonitoredConsumer; +import org.apache.maven.archiva.consumers.ConsumerException; +import org.apache.maven.archiva.consumers.InvalidRepositoryContentConsumer; +import org.apache.maven.archiva.model.ArchivaRepository; + +import java.util.List; + +/** + * InvalidScanConsumer + * + * @author Joakim Erdfelt + * @version $Id$ + */ +public class InvalidScanConsumer + extends AbstractMonitoredConsumer + implements InvalidRepositoryContentConsumer +{ + private int processCount = 0; + + public void beginScan( ArchivaRepository repository ) + throws ConsumerException + { + /* do nothing */ + } + + public void completeScan() + { + /* do nothing */ + } + + public List getExcludes() + { + return null; + } + + public List getIncludes() + { + return null; + } + + public void processFile( String path ) + throws ConsumerException + { + processCount++; + } + + public String getDescription() + { + return "Bad Content Scan Consumer (for testing)"; + } + + public String getId() + { + return "test-invalid-consumer"; + } + + public boolean isPermanent() + { + return false; + } + + public int getProcessCount() + { + return processCount; + } + + public void setProcessCount( int processCount ) + { + this.processCount = processCount; + } +} diff --git a/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/ScanConsumer.java b/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/KnownScanConsumer.java similarity index 87% rename from archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/ScanConsumer.java rename to archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/KnownScanConsumer.java index 8ab2012f7..d16b6aa61 100644 --- a/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/ScanConsumer.java +++ b/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/KnownScanConsumer.java @@ -22,7 +22,6 @@ package org.apache.maven.archiva.repository.scanner; import org.apache.maven.archiva.consumers.AbstractMonitoredConsumer; import org.apache.maven.archiva.consumers.ConsumerException; import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer; -import org.apache.maven.archiva.consumers.RepositoryContentConsumer; import org.apache.maven.archiva.model.ArchivaRepository; import java.util.ArrayList; @@ -35,7 +34,9 @@ import java.util.List; * @author Joakim Erdfelt * @version $Id$ */ -public class ScanConsumer extends AbstractMonitoredConsumer implements KnownRepositoryContentConsumer +public class KnownScanConsumer + extends AbstractMonitoredConsumer + implements KnownRepositoryContentConsumer { private int processCount = 0; @@ -67,12 +68,14 @@ public class ScanConsumer extends AbstractMonitoredConsumer implements KnownRepo return "Scan Consumer (for testing)"; } - public void beginScan( ArchivaRepository repository ) throws ConsumerException + public void beginScan( ArchivaRepository repository ) + throws ConsumerException { /* do nothing */ } - public void processFile( String path ) throws ConsumerException + public void processFile( String path ) + throws ConsumerException { this.processCount++; } diff --git a/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerTest.java b/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerTest.java index c85c39758..305096163 100644 --- a/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerTest.java +++ b/archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerTest.java @@ -27,6 +27,7 @@ import org.codehaus.plexus.PlexusTestCase; import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -72,7 +73,7 @@ public class RepositoryScannerTest return repo; } - + private void assertMinimumHits( String msg, int minimumHitCount, long actualCount ) { if ( actualCount < minimumHitCount ) @@ -82,27 +83,54 @@ public class RepositoryScannerTest } } + private RepositoryScanner lookupRepositoryScanner() + throws Exception + { + return (RepositoryScanner) lookup( RepositoryScanner.class ); + } + + private List getIgnoreList() + { + List ignores = new ArrayList(); + ignores.addAll( Arrays.asList( RepositoryScanner.IGNORABLE_CONTENT ) ); + return ignores; + } + public void testDefaultRepositoryScanner() - throws RepositoryException + throws Exception { ArchivaRepository repository = createDefaultRepository(); - List consumers = new ArrayList(); - ScanConsumer consumer = new ScanConsumer(); - consumer.setIncludes( new String[] { "**/*.jar" } ); - consumers.add( consumer ); - - RepositoryScanner scanner = new RepositoryScanner(); - boolean includeSnapshots = true; - RepositoryContentStatistics stats = scanner.scan( repository, consumers, includeSnapshots ); + List knownConsumers = new ArrayList(); + KnownScanConsumer consumer = new KnownScanConsumer(); + consumer.setIncludes( new String[] { + "**/*.jar", + "**/*.war", + "**/*.pom", + "**/maven-metadata.xml", + "**/*-site.xml", + "**/*.zip", + "**/*.tar.gz", + "**/*.sha1", + "**/*.md5" } ); + knownConsumers.add( consumer ); + + List invalidConsumers = new ArrayList(); + InvalidScanConsumer badconsumer = new InvalidScanConsumer(); + invalidConsumers.add( badconsumer ); + + RepositoryScanner scanner = lookupRepositoryScanner(); + RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers, + getIgnoreList(), RepositoryScanner.FRESH_SCAN ); assertNotNull( "Stats should not be null.", stats ); assertMinimumHits( "Stats.totalFileCount", 17, stats.getTotalFileCount() ); assertMinimumHits( "Processed Count", 17, consumer.getProcessCount() ); + assertEquals( "Processed Count (of invalid items)", 6, badconsumer.getProcessCount() ); } public void testDefaultRepositoryArtifactScanner() - throws RepositoryException + throws Exception { List actualArtifactPaths = new ArrayList(); @@ -140,14 +168,18 @@ public class RepositoryScannerTest ArchivaRepository repository = createDefaultRepository(); - List consumers = new ArrayList(); - ScanConsumer consumer = new ScanConsumer(); + List knownConsumers = new ArrayList(); + KnownScanConsumer consumer = new KnownScanConsumer(); consumer.setIncludes( ARTIFACT_PATTERNS ); - consumers.add( consumer ); + knownConsumers.add( consumer ); + + List invalidConsumers = new ArrayList(); + InvalidScanConsumer badconsumer = new InvalidScanConsumer(); + invalidConsumers.add( badconsumer ); - RepositoryScanner scanner = new RepositoryScanner(); - boolean includeSnapshots = true; - RepositoryContentStatistics stats = scanner.scan( repository, consumers, includeSnapshots ); + RepositoryScanner scanner = lookupRepositoryScanner(); + RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers, + getIgnoreList(), RepositoryScanner.FRESH_SCAN ); assertNotNull( "Stats should not be null.", stats ); assertMinimumHits( "Stats.totalFileCount", actualArtifactPaths.size(), stats.getTotalFileCount() ); @@ -155,7 +187,7 @@ public class RepositoryScannerTest } public void testDefaultRepositoryMetadataScanner() - throws RepositoryException + throws Exception { List actualMetadataPaths = new ArrayList(); @@ -172,22 +204,26 @@ public class RepositoryScannerTest ArchivaRepository repository = createDefaultRepository(); - List consumers = new ArrayList(); - ScanConsumer consumer = new ScanConsumer(); - consumer.setIncludes( new String[] { "**/maven-metadata*.xml" } ); - consumers.add( consumer ); + List knownConsumers = new ArrayList(); + KnownScanConsumer knownConsumer = new KnownScanConsumer(); + knownConsumer.setIncludes( new String[] { "**/maven-metadata*.xml" } ); + knownConsumers.add( knownConsumer ); - RepositoryScanner scanner = new RepositoryScanner(); - boolean includeSnapshots = true; - RepositoryContentStatistics stats = scanner.scan( repository, consumers, includeSnapshots ); + List invalidConsumers = new ArrayList(); + InvalidScanConsumer badconsumer = new InvalidScanConsumer(); + invalidConsumers.add( badconsumer ); + + RepositoryScanner scanner = lookupRepositoryScanner(); + RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers, + getIgnoreList(), RepositoryScanner.FRESH_SCAN ); assertNotNull( "Stats should not be null.", stats ); assertMinimumHits( "Stats.totalFileCount", actualMetadataPaths.size(), stats.getTotalFileCount() ); - assertMinimumHits( "Processed Count", actualMetadataPaths.size(), consumer.getProcessCount() ); + assertMinimumHits( "Processed Count", actualMetadataPaths.size(), knownConsumer.getProcessCount() ); } public void testDefaultRepositoryProjectScanner() - throws RepositoryException + throws Exception { List actualProjectPaths = new ArrayList(); @@ -204,14 +240,18 @@ public class RepositoryScannerTest ArchivaRepository repository = createDefaultRepository(); - List consumers = new ArrayList(); - ScanConsumer consumer = new ScanConsumer(); + List knownConsumers = new ArrayList(); + KnownScanConsumer consumer = new KnownScanConsumer(); consumer.setIncludes( new String[] { "**/*.pom" } ); - consumers.add( consumer ); + knownConsumers.add( consumer ); + + List invalidConsumers = new ArrayList(); + InvalidScanConsumer badconsumer = new InvalidScanConsumer(); + invalidConsumers.add( badconsumer ); - RepositoryScanner scanner = new RepositoryScanner(); - boolean includeSnapshots = true; - RepositoryContentStatistics stats = scanner.scan( repository, consumers, includeSnapshots ); + RepositoryScanner scanner = lookupRepositoryScanner(); + RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers, + getIgnoreList(), RepositoryScanner.FRESH_SCAN ); assertNotNull( "Stats should not be null.", stats ); assertMinimumHits( "Stats.totalFileCount", actualProjectPaths.size(), stats.getTotalFileCount() ); @@ -219,7 +259,7 @@ public class RepositoryScannerTest } public void testLegacyRepositoryArtifactScanner() - throws RepositoryException + throws Exception { List actualArtifactPaths = new ArrayList(); @@ -240,14 +280,18 @@ public class RepositoryScannerTest ArchivaRepository repository = createLegacyRepository(); - List consumers = new ArrayList(); - ScanConsumer consumer = new ScanConsumer(); + List knownConsumers = new ArrayList(); + KnownScanConsumer consumer = new KnownScanConsumer(); consumer.setIncludes( ARTIFACT_PATTERNS ); - consumers.add( consumer ); + knownConsumers.add( consumer ); + + List invalidConsumers = new ArrayList(); + InvalidScanConsumer badconsumer = new InvalidScanConsumer(); + invalidConsumers.add( badconsumer ); - RepositoryScanner scanner = new RepositoryScanner(); - boolean includeSnapshots = true; - RepositoryContentStatistics stats = scanner.scan( repository, consumers, includeSnapshots ); + RepositoryScanner scanner = lookupRepositoryScanner(); + RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers, + getIgnoreList(), RepositoryScanner.FRESH_SCAN ); assertNotNull( "Stats should not be null.", stats ); assertMinimumHits( "Stats.totalFileCount", actualArtifactPaths.size(), stats.getTotalFileCount() ); diff --git a/archiva-cli/src/main/java/org/apache/maven/archiva/cli/ArchivaCli.java b/archiva-cli/src/main/java/org/apache/maven/archiva/cli/ArchivaCli.java index e5237d264..3f5b03872 100644 --- a/archiva-cli/src/main/java/org/apache/maven/archiva/cli/ArchivaCli.java +++ b/archiva-cli/src/main/java/org/apache/maven/archiva/cli/ArchivaCli.java @@ -44,6 +44,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -70,7 +71,7 @@ public class ArchivaCli public static final char CONSUMERS = 'u'; public static final char LIST_CONSUMERS = 'l'; - + public static final char DUMP_CONFIGURATION = 'd'; // ---------------------------------------------------------------------------- @@ -89,7 +90,7 @@ public class ArchivaCli * @plexus.requirement */ private ArchivaConfiguration archivaConfiguration; - + public static void main( String[] args ) throws Exception { @@ -134,7 +135,7 @@ public class ArchivaCli Option dumpConfig = createOption( DUMP_CONFIGURATION, "dumpconfig", 0, "Dump Current Configuration." ); options.addOption( dumpConfig ); - + return options; } @@ -170,15 +171,21 @@ public class ArchivaCli ArchivaRepository repo = new ArchivaRepository( "cliRepo", "Archiva CLI Provided Repo", "file://" + path ); - List consumerList = new ArrayList(); + List knownConsumerList = new ArrayList(); + + knownConsumerList.addAll( getConsumerList( cli, plexus ) ); - consumerList.addAll( getConsumerList( cli, plexus ) ); + List invalidConsumerList = Collections.EMPTY_LIST; - RepositoryScanner scanner = new RepositoryScanner(); + List ignoredContent = new ArrayList(); + ignoredContent.addAll( Arrays.asList( RepositoryScanner.IGNORABLE_CONTENT ) ); + + RepositoryScanner scanner = (RepositoryScanner) plexus.lookup( RepositoryScanner.class ); try { - RepositoryContentStatistics stats = scanner.scan( repo, consumerList, true ); + RepositoryContentStatistics stats = scanner.scan( repo, knownConsumerList, invalidConsumerList, + ignoredContent, RepositoryScanner.FRESH_SCAN ); SimpleDateFormat df = new SimpleDateFormat(); System.out.println( "" ); @@ -186,8 +193,8 @@ public class ArchivaCli System.out.println( " Repository URL : " + repo.getUrl() ); System.out.println( " Repository Name : " + repo.getModel().getName() ); System.out.println( " Repository Layout : " + repo.getModel().getLayoutName() ); - System.out.println( " Consumers : (" + consumerList.size() + " active)" ); - for ( Iterator iter = consumerList.iterator(); iter.hasNext(); ) + System.out.println( " Consumers : (" + knownConsumerList.size() + " active)" ); + for ( Iterator iter = knownConsumerList.iterator(); iter.hasNext(); ) { RepositoryContentConsumer consumer = (RepositoryContentConsumer) iter.next(); System.out.println( " " + consumer.getId() + " - " + consumer.getDescription() ); @@ -296,11 +303,13 @@ public class ArchivaCli showFatalError( "Error converting repository.", e, true ); } } - - private void dumpConfiguration( PlexusContainer plexus ) throws ComponentLookupException - { + + private void dumpConfiguration( PlexusContainer plexus ) + throws ComponentLookupException + { archivaConfiguration = (ArchivaConfiguration) plexus.lookup( ArchivaConfiguration.ROLE, "cli" ); - - System.out.println( "File Type Count: " + archivaConfiguration.getConfiguration().getRepositoryScanning().getFileTypes().size() ); + + System.out.println( "File Type Count: " + + archivaConfiguration.getConfiguration().getRepositoryScanning().getFileTypes().size() ); } } diff --git a/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaScheduledTaskExecutor.java b/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaScheduledTaskExecutor.java index 3caadfc48..cf931977d 100644 --- a/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaScheduledTaskExecutor.java +++ b/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaScheduledTaskExecutor.java @@ -20,16 +20,15 @@ package org.apache.maven.archiva.scheduled.executors; */ import org.apache.commons.collections.CollectionUtils; -import org.apache.maven.archiva.configuration.ArchivaConfiguration; -import org.apache.maven.archiva.configuration.RepositoryScanningConfiguration; +import org.apache.maven.archiva.common.utils.DateUtil; import org.apache.maven.archiva.database.ArchivaDAO; import org.apache.maven.archiva.database.ArchivaDatabaseException; import org.apache.maven.archiva.database.RepositoryDAO; +import org.apache.maven.archiva.database.constraints.MostRecentRepositoryScanStatistics; import org.apache.maven.archiva.database.updater.DatabaseUpdater; import org.apache.maven.archiva.model.ArchivaRepository; import org.apache.maven.archiva.model.RepositoryContentStatistics; import org.apache.maven.archiva.repository.RepositoryException; -import org.apache.maven.archiva.repository.scanner.RepositoryContentConsumerUtil; import org.apache.maven.archiva.repository.scanner.RepositoryScanner; import org.apache.maven.archiva.scheduled.tasks.DatabaseTask; import org.apache.maven.archiva.scheduled.tasks.RepositoryTask; @@ -38,9 +37,7 @@ import org.codehaus.plexus.taskqueue.Task; import org.codehaus.plexus.taskqueue.execution.TaskExecutionException; import org.codehaus.plexus.taskqueue.execution.TaskExecutor; -import java.util.ArrayList; import java.util.List; -import java.util.Map; /** * @@ -54,13 +51,6 @@ public class ArchivaScheduledTaskExecutor extends AbstractLogEnabled implements TaskExecutor { - /** - * Configuration store. - * - * @plexus.requirement - */ - private ArchivaConfiguration archivaConfiguration; - /** * @plexus.requirement role-hint="jdo" */ @@ -77,17 +67,11 @@ public class ArchivaScheduledTaskExecutor private RepositoryDAO repositoryDAO; /** - * The collection of available database consumers. - * @plexus.requirement role="org.apache.maven.archiva.consumers.ArchivaArtifactConsumer" - */ - private Map availableDBConsumers; - - /** - * The collection of available repository consumers. + * The repository scanner component. * * @plexus.requirement */ - private RepositoryContentConsumerUtil repositoryContentConsumerUtil; + private RepositoryScanner repoScanner; public void executeTask( Task task ) throws TaskExecutionException @@ -149,13 +133,21 @@ public class ArchivaScheduledTaskExecutor { ArchivaRepository arepo = repositoryDAO.getRepository( task.getRepositoryId() ); - RepositoryScanner scanner = new RepositoryScanner(); + long sinceWhen = RepositoryScanner.FRESH_SCAN; + + List results = dao.query( new MostRecentRepositoryScanStatistics( arepo.getId() ) ); + + if ( CollectionUtils.isNotEmpty( results ) ) + { + RepositoryContentStatistics lastStats = (RepositoryContentStatistics) results.get( 0 ); + sinceWhen = lastStats.getWhenGathered().getTime() + lastStats.getDuration(); + } - RepositoryContentStatistics stats = scanner.scan( arepo, getActiveConsumerList(), true ); + RepositoryContentStatistics stats = repoScanner.scan( arepo, sinceWhen ); dao.save( stats ); - getLogger().info( "Finished repository task: " + stats.getDuration() + " ms." ); + getLogger().info( "Finished repository task: " + DateUtil.getDuration( stats.getDuration() ) + "." ); } catch ( ArchivaDatabaseException e ) { @@ -166,25 +158,4 @@ public class ArchivaScheduledTaskExecutor throw new TaskExecutionException( "Repository error when executing repository job.", e ); } } - - private List getActiveConsumerList() - { - List activeConsumers = new ArrayList(); - - RepositoryScanningConfiguration repoScanningConfig = archivaConfiguration.getConfiguration() - .getRepositoryScanning(); - - List configuredGoodConsumers = new ArrayList(); - List configuredBadConsumers = new ArrayList(); - - configuredGoodConsumers.addAll( CollectionUtils.select( repoScanningConfig.getGoodConsumers(), - repositoryContentConsumerUtil - .getKnownSelectionPredicate() ) ); - - configuredBadConsumers.addAll( CollectionUtils.select( repoScanningConfig.getBadConsumers(), - repositoryContentConsumerUtil - .getInvalidSelectionPredicate() ) ); - - return activeConsumers; - } } diff --git a/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/admin/scanning/RepositoryScanningAction.java b/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/admin/scanning/RepositoryScanningAction.java index c326b5355..b8035c7a5 100644 --- a/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/admin/scanning/RepositoryScanningAction.java +++ b/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/admin/scanning/RepositoryScanningAction.java @@ -34,6 +34,7 @@ import org.codehaus.plexus.security.ui.web.interceptor.SecureActionException; import org.codehaus.plexus.xwork.action.PlexusActionSupport; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -55,10 +56,16 @@ public class RepositoryScanningAction private ArchivaConfiguration archivaConfiguration; private Map fileTypeMap; + + private List fileTypeIds; private List goodConsumers = new ArrayList(); private List badConsumers = new ArrayList(); + + private String pattern; + + private String fileTypeId; public void prepare() throws Exception @@ -74,6 +81,32 @@ public class RepositoryScanningAction badConsumers.clear(); badConsumers.addAll( config.getRepositoryScanning().getBadConsumers() ); + + fileTypeIds = new ArrayList(); + fileTypeIds.addAll( fileTypeMap.keySet() ); + Collections.sort( fileTypeIds ); + } + + public String removeFiletypePattern() + { + getLogger().info( "Remove File Type Pattern [" + getFileTypeId() + ":" + getPattern() + "]" ); + + // TODO: remove the filetype + // TODO: save configuration + + return INPUT; + } + + public String addFiletypePattern() + { + getLogger().info( "Add New File Type Pattern [" + getFileTypeId() + ":" + getPattern() + "]" ); + + // TODO: add the filetype. + // TODO: report error if filetype pattern already exists. + // TODO: report success (message) if added successfully. + // TODO: save configuration each time. + + return INPUT; } public SecureActionBundle getSecureActionBundle() @@ -92,28 +125,38 @@ public class RepositoryScanningAction return badConsumers; } - public void setBadConsumers( List badConsumers ) + public Map getFileTypeMap() { - this.badConsumers = badConsumers; + return fileTypeMap; } - public Map getFileTypeMap() + public List getGoodConsumers() { - return fileTypeMap; + return goodConsumers; } - public void setFileTypeMap( Map fileTypeMap ) + public String getFileTypeId() { - this.fileTypeMap = fileTypeMap; + return fileTypeId; } - public List getGoodConsumers() + public void setFileTypeId( String fileTypeId ) { - return goodConsumers; + this.fileTypeId = fileTypeId; + } + + public String getPattern() + { + return pattern; + } + + public void setPattern( String pattern ) + { + this.pattern = pattern; } - public void setGoodConsumers( List goodConsumers ) + public List getFileTypeIds() { - this.goodConsumers = goodConsumers; + return fileTypeIds; } } diff --git a/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/repositoryScanning.jsp b/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/repositoryScanning.jsp index 3814dc94e..039ac84f2 100644 --- a/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/repositoryScanning.jsp +++ b/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/repositoryScanning.jsp @@ -17,15 +17,16 @@ ~ under the License. --%> -<%@ taglib prefix="ww" uri="/webwork" %> -<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> -<%@ taglib prefix="pss" uri="/plexusSecuritySystem" %> -<%@ taglib prefix="archiva" uri="http://maven.apache.org/archiva" %> +<%@ taglib prefix="ww" uri="/webwork"%> +<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%> +<%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %> +<%@ taglib prefix="pss" uri="/plexusSecuritySystem"%> +<%@ taglib prefix="archiva" uri="http://maven.apache.org/archiva"%> - Administration - Repository Scanning - +Administration - Repository Scanning + @@ -34,11 +35,40 @@
- - + + + + + + + + +
-

Repository Scanning - File Types

+

Repository Scanning - File Types

@@ -47,46 +77,75 @@ <%-- Display the filetypes. --%> - - - - - - - - - - - -
- -
- <%-- Does this even make sense for file types? --%> -
- -

${filetype.key}

+ + + + + + + + + + +
+ +
<%-- Does this even make sense for file types? --%>
+ +

${filetypeId}

- + + + + + + + + + + + + + + + + + -
+ ${escapedPattern} + + + + +
- ${pattern} + - " /> + + +
- -
- - + +
+ + -

Repository Scanning - Consumers of Good Content

- +

Repository Scanning - Consumers of Good Content

+ <%-- No Good Consumers. Eeek! --%> @@ -94,33 +153,31 @@ <%-- Display the consumers. --%> - - - - - - - - - - - - - - - - - -
${consumer} - " /> -
- + + + + + + + + + + + + + + + + + +
${consumer}" />
+ - -

Repository Scanning - Consumers of Bad Content

- + +

Repository Scanning - Consumers of Bad Content

+ <%-- No Bad Consumers. Eeek! --%> @@ -128,32 +185,26 @@ <%-- Display the consumers. --%> - - - - - - - - - - - - - - - - - -
${consumer} - " /> -
- - - - -
+ + + + + + + + + + + + + + + +
${consumer}" />
+ + +
diff --git a/archiva-web/archiva-webapp/src/main/webapp/css/site.css b/archiva-web/archiva-webapp/src/main/webapp/css/site.css index 5091385f1..587dd7161 100644 --- a/archiva-web/archiva-webapp/src/main/webapp/css/site.css +++ b/archiva-web/archiva-webapp/src/main/webapp/css/site.css @@ -307,4 +307,22 @@ div.admin div.dark { div.admin div.controls { float: right; font-size: 8pt !important; -} \ No newline at end of file +} + +div.admin div.filetype table { + margin-left: 25px; + border: 1px solid gray; +} + +div.filetype table td.controls { + width: 5%; +} + +div.filetype table td.odd { + background-color: #dddddd; +} + +div.filetype table td.event { + background-color: white; +} + -- 2.39.5