Refactoring of test cases.

git-svn-id: https://svn.apache.org/repos/asf/maven/archiva/trunk@512272 13f79535-47bb-0310-9956-ffa450edef68

archiva-repository-layer/pom.xml

@@ -58,6 +58,10 @@
       <groupId>org.apache.maven</groupId>
       <artifactId>maven-repository-metadata</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+    </dependency>
   </dependencies>
   <build>
     <plugins>

archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/DefinedRepositories.java

@@ -35,35 +35,35 @@ public interface DefinedRepositories
      * @return the list of repositories.
      */
     public List getAllRepositories();
-    
+
     /**
      * Get the list of managed (local) repositories.
      * 
      * @return the list of managed (local) repositories.
      */
     public List getManagedRepositories();
-    
+
     /**
      * Get the list of remote repositories.
      * 
      * @return the list of remote repositories.
      */
     public List getRemoteRepositories();
-    
+
     /**
      * Add a repository.
      * 
      * @param repository the repository to add.
      */
-    public void addRepository(Repository repository);
-    
+    public void addRepository( Repository repository );
+
     /**
      * Remove a repository.
      * 
      * @param repository the repository to add.
      */
-    public void removeRepository(Repository repository);
-    
+    public void removeRepository( Repository repository );
+
     /**
      * Get a repository using the provided repository key.
      *  

archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScanner.java

@@ -0,0 +1,196 @@
+package org.apache.maven.archiva.repository.scanner;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.archiva.common.consumers.Consumer;
+import org.apache.maven.archiva.repository.RepositoryException;
+import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.codehaus.plexus.util.DirectoryWalker;
+import org.codehaus.plexus.util.FileUtils;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * RepositoryScanner 
+ *
+ * @author <a href="mailto:joakim@erdfelt.com">Joakim Erdfelt</a>
+ * @version $Id$
+ */
+public class RepositoryScanner
+{
+    /**
+     * Standard patterns to exclude from discovery as they are usually noise.
+     */
+    private static final String[] STANDARD_SCANNER_EXCLUDES = {
+        "bin/**",
+        "reports/**",
+        ".index",
+        ".reports/**",
+        ".maven/**",
+        "**/*snapshot-version",
+        "*/website/**",
+        "*/licences/**",
+        "**/.htaccess",
+        "**/*.html",
+        "**/*.txt",
+        "**/README*",
+        "**/CHANGELOG*",
+        "**/KEYS*" };
+
+    /**
+     * Walk the repository, and report to the consumers the files found.
+     * 
+     * Report changes to the appropriate Consumer.
+     * 
+     * This is just a convenience method to {@link #scan(ArtifactRepository, List, boolean, long, List, List)}
+     * equivalent to calling <code>scan( repository, consumers, includeSnapshots, 0, null, null );</code>
+     * 
+     * @param repository the repository to change.
+     * @param consumers use the provided list of consumers.
+     * @param includeSnapshots true to include snapshots in the walking of this repository.
+     * @return the statistics for this scan.
+     * @throws RepositoryException if there was a fundamental problem with getting the discoverer started.
+     */
+    public ScanStatistics scan( ArtifactRepository repository, List consumers, boolean includeSnapshots )
+        throws RepositoryException
+    {
+        return scan( repository, consumers, includeSnapshots, 0, null, null );
+    }
+
+    /**
+     * Walk the repository, and report to the consumers the files found.
+     * 
+     * Report changes to the appropriate Consumer.
+     * 
+     * @param repository the repository to change.
+     * @param consumers use the provided list of consumers.
+     * @param includeSnapshots true to include snapshots in the scanning of this repository.
+     * @param onlyModifiedAfterTimestamp Only report to the consumers, files that have a {@link File#lastModified()}) 
+     *          after the provided timestamp.
+     * @param extraFileExclusions an optional list of file exclusions on the walk.
+     * @param extraFileInclusions an optional list of file inclusions on the walk.
+     * @return the statistics for this scan.
+     * @throws RepositoryException if there was a fundamental problem with getting the discoverer started. 
+     */
+    public ScanStatistics scan( ArtifactRepository repository, List consumers, boolean includeSnapshots,
+                                long onlyModifiedAfterTimestamp, List extraFileExclusions, List extraFileInclusions )
+        throws RepositoryException
+    {
+        if ( repository == null )
+        {
+            throw new IllegalArgumentException( "Unable to operate on a null repository." );
+        }
+
+        if ( !"file".equals( repository.getProtocol() ) )
+        {
+            throw new UnsupportedOperationException( "Only filesystem repositories are supported." );
+        }
+
+        File repositoryBase = new File( repository.getBasedir() );
+
+        if ( !repositoryBase.exists() )
+        {
+            throw new UnsupportedOperationException( "Unable to scan a repository, directory "
+                + repositoryBase.getAbsolutePath() + " does not exist." );
+        }
+
+        if ( !repositoryBase.isDirectory() )
+        {
+            throw new UnsupportedOperationException( "Unable to scan a repository, path "
+                + repositoryBase.getAbsolutePath() + " is not a directory." );
+        }
+
+        // Setup Includes / Excludes.
+
+        List allExcludes = new ArrayList();
+        List allIncludes = new ArrayList();
+
+        // Exclude all of the SCM patterns.
+        allExcludes.addAll( FileUtils.getDefaultExcludesAsList() );
+
+        // Exclude all of the archiva noise patterns.
+        allExcludes.addAll( Arrays.asList( STANDARD_SCANNER_EXCLUDES ) );
+
+        if ( !includeSnapshots )
+        {
+            allExcludes.add( "**/*-SNAPSHOT*" );
+        }
+
+        if ( extraFileExclusions != null )
+        {
+            allExcludes.addAll( extraFileExclusions );
+        }
+
+        Iterator it = consumers.iterator();
+        while ( it.hasNext() )
+        {
+            Consumer consumer = (Consumer) it.next();
+
+            /* NOTE: Do not insert the consumer exclusion patterns here.
+             * Exclusion patterns are handled by RepositoryScanner.wantsFile(Consumer, String)
+             * 
+             * addUniqueElements( consumer.getExcludePatterns(), allExcludes );
+             */
+            addUniqueElements( consumer.getIncludePatterns(), allIncludes );
+        }
+
+        if ( extraFileInclusions != null )
+        {
+            allIncludes.addAll( extraFileInclusions );
+        }
+
+        // Setup Directory Walker
+
+        DirectoryWalker dirWalker = new DirectoryWalker();
+
+        dirWalker.setBaseDir( repositoryBase );
+
+        dirWalker.setIncludes( allIncludes );
+        dirWalker.setExcludes( allExcludes );
+
+        // Setup the Scan Instance
+        RepositoryScannerInstance scannerInstance = new RepositoryScannerInstance( repository, consumers );
+        scannerInstance.setOnlyModifiedAfterTimestamp( onlyModifiedAfterTimestamp );
+
+        dirWalker.addDirectoryWalkListener( scannerInstance );
+
+        // Execute scan.
+        dirWalker.scan();
+
+        return scannerInstance.getStatistics();
+    }
+    
+    private void addUniqueElements( List fromList, List toList )
+    {
+        Iterator itFrom = fromList.iterator();
+        while ( itFrom.hasNext() )
+        {
+            Object o = itFrom.next();
+            if ( !toList.contains( o ) )
+            {
+                toList.add( o );
+            }
+        }
+    }
+}

archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerInstance.java

@@ -0,0 +1,197 @@
+package org.apache.maven.archiva.repository.scanner;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.commons.lang.SystemUtils;
+import org.apache.maven.archiva.common.consumers.Consumer;
+import org.apache.maven.archiva.common.utils.BaseFile;
+import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.codehaus.plexus.util.DirectoryWalkListener;
+import org.codehaus.plexus.util.SelectorUtils;
+import org.codehaus.plexus.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * RepositoryScannerInstance 
+ *
+ * @author <a href="mailto:joakim@erdfelt.com">Joakim Erdfelt</a>
+ * @version $Id$
+ */
+public class RepositoryScannerInstance implements DirectoryWalkListener
+{
+    private static Logger log = LoggerFactory.getLogger( RepositoryScannerInstance.class );
+
+    private List consumers;
+
+    private ArtifactRepository repository;
+
+    private boolean isCaseSensitive = true;
+
+    private ScanStatistics stats;
+
+    private long onlyModifiedAfterTimestamp = 0;
+
+    public RepositoryScannerInstance( ArtifactRepository repository, List consumerList )
+    {
+        this.repository = repository;
+        this.consumers = consumerList;
+        stats = new ScanStatistics( repository );
+
+        Iterator it = this.consumers.iterator();
+        while ( it.hasNext() )
+        {
+            Consumer consumer = (Consumer) it.next();
+
+            if ( !consumer.init( this.repository ) )
+            {
+                throw new IllegalStateException( "Consumer [" + consumer.getName() +
+                    "] is reporting that it is incompatible with the [" + repository.getId() + "] repository." );
+            }
+        }
+
+        if ( SystemUtils.IS_OS_WINDOWS )
+        {
+            isCaseSensitive = false;
+        }
+    }
+
+    public ScanStatistics getStatistics()
+    {
+        return stats;
+    }
+
+    public void directoryWalkStarting( File basedir )
+    {
+        log.info( "Walk Started: [" + this.repository.getId() + "] " + this.repository.getBasedir() );
+        stats.reset();
+        stats.timestampStarted = System.currentTimeMillis();
+    }
+
+    public void directoryWalkStep( int percentage, File file )
+    {
+        log.debug( "Walk Step: " + percentage + ", " + file );
+
+        // Timestamp finished points to the last successful scan, not this current one.
+        if ( file.lastModified() < onlyModifiedAfterTimestamp )
+        {
+            // Skip file as no change has occured.
+            log.debug( "Skipping, No Change: " + file.getAbsolutePath() );
+            stats.filesSkipped++;
+            return;
+        }
+
+        synchronized ( consumers )
+        {
+            stats.filesIncluded++;
+
+            BaseFile basefile = new BaseFile( repository.getBasedir(), file );
+
+            Iterator itConsumers = this.consumers.iterator();
+            while ( itConsumers.hasNext() )
+            {
+                Consumer consumer = (Consumer) itConsumers.next();
+
+                if ( wantsFile( consumer, StringUtils.replace( basefile.getRelativePath(), "\\", "/" ) ) )
+                {
+                    try
+                    {
+                        log.debug( "Sending to consumer: " + consumer.getName() );
+                        stats.filesConsumed++;
+                        consumer.processFile( basefile );
+                    }
+                    catch ( Exception e )
+                    {
+                        /* Intentionally Catch all exceptions.
+                         * So that the discoverer processing can continue.
+                         */
+                        log.error( "Consumer [" + consumer.getName() + "] had an error when processing file [" +
+                            basefile.getAbsolutePath() + "]: " + e.getMessage(), e );
+                    }
+                }
+                else
+                {
+                    log.debug(
+                        "Skipping consumer " + consumer.getName() + " for file " + basefile.getRelativePath() );
+                }
+            }
+        }
+    }
+
+    public void directoryWalkFinished()
+    {
+        log.info( "Walk Finished: [" + this.repository.getId() + "] " + this.repository.getBasedir() );
+        stats.timestampFinished = System.currentTimeMillis();
+    }
+
+    private boolean wantsFile( Consumer consumer, String relativePath )
+    {
+        Iterator it;
+
+        // Test excludes first.
+        it = consumer.getExcludePatterns().iterator();
+        while ( it.hasNext() )
+        {
+            String pattern = (String) it.next();
+            if ( SelectorUtils.matchPath( pattern, relativePath, isCaseSensitive ) )
+            {
+                // Definately does NOT WANT FILE.
+                return false;
+            }
+        }
+
+        // Now test includes.
+        it = consumer.getIncludePatterns().iterator();
+        while ( it.hasNext() )
+        {
+            String pattern = (String) it.next();
+            if ( SelectorUtils.matchPath( pattern, relativePath, isCaseSensitive ) )
+            {
+                // Specifically WANTS FILE.
+                return true;
+            }
+        }
+
+        // Not included, and Not excluded?  Default to EXCLUDE.
+        return false;
+    }
+
+    public long getOnlyModifiedAfterTimestamp()
+    {
+        return onlyModifiedAfterTimestamp;
+    }
+
+    public void setOnlyModifiedAfterTimestamp( long onlyModifiedAfterTimestamp )
+    {
+        this.onlyModifiedAfterTimestamp = onlyModifiedAfterTimestamp;
+    }
+
+    /**
+     * Debug method from DirectoryWalker.
+     */
+    public void debug( String message )
+    {
+        log.debug( "Repository Scanner: " + message );
+    }
+}

archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/ScanStatistics.java

@@ -0,0 +1,198 @@
+package org.apache.maven.archiva.repository.scanner;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.codehaus.plexus.logging.Logger;
+import org.codehaus.plexus.util.IOUtil;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Properties;
+
+/**
+ * ScanStatistics 
+ *
+ * @author <a href="mailto:joakim@erdfelt.com">Joakim Erdfelt</a>
+ * @version $Id$
+ */
+public class ScanStatistics
+{
+    private static final String PROP_FILES_CONSUMED = "scan.consumed.files";
+
+    private static final String PROP_FILES_INCLUDED = "scan.included.files";
+
+    private static final String PROP_FILES_SKIPPED = "scan.skipped.files";
+
+    private static final String PROP_TIMESTAMP_STARTED = "scan.started.timestamp";
+
+    private static final String PROP_TIMESTAMP_FINISHED = "scan.finished.timestamp";
+
+    protected long timestampStarted = 0;
+
+    protected long timestampFinished = 0;
+
+    protected long filesIncluded = 0;
+
+    protected long filesConsumed = 0;
+
+    protected long filesSkipped = 0;
+
+    private ArtifactRepository repository;
+
+    public ScanStatistics( ArtifactRepository repository )
+    {
+        this.repository = repository;
+    }
+
+    public void load( String filename )
+        throws IOException
+    {
+        File repositoryBase = new File( this.repository.getBasedir() );
+
+        File scanProperties = new File( repositoryBase, filename );
+        FileInputStream fis = null;
+        try
+        {
+            Properties props = new Properties();
+            fis = new FileInputStream( scanProperties );
+            props.load( fis );
+
+            timestampFinished = NumberUtils.toLong( props.getProperty( PROP_TIMESTAMP_FINISHED ), 0 );
+            timestampStarted = NumberUtils.toLong( props.getProperty( PROP_TIMESTAMP_STARTED ), 0 );
+            filesIncluded = NumberUtils.toLong( props.getProperty( PROP_FILES_INCLUDED ), 0 );
+            filesConsumed = NumberUtils.toLong( props.getProperty( PROP_FILES_CONSUMED ), 0 );
+            filesSkipped = NumberUtils.toLong( props.getProperty( PROP_FILES_SKIPPED ), 0 );
+        }
+        catch ( IOException e )
+        {
+            reset();
+            throw e;
+        }
+        finally
+        {
+            IOUtil.close( fis );
+        }
+    }
+
+    public void save( String filename )
+        throws IOException
+    {
+        Properties props = new Properties();
+        props.setProperty( PROP_TIMESTAMP_FINISHED, String.valueOf( timestampFinished ) );
+        props.setProperty( PROP_TIMESTAMP_STARTED, String.valueOf( timestampStarted ) );
+        props.setProperty( PROP_FILES_INCLUDED, String.valueOf( filesIncluded ) );
+        props.setProperty( PROP_FILES_CONSUMED, String.valueOf( filesConsumed ) );
+        props.setProperty( PROP_FILES_SKIPPED, String.valueOf( filesSkipped ) );
+
+        File repositoryBase = new File( this.repository.getBasedir() );
+        File statsFile = new File( repositoryBase, filename );
+
+        FileOutputStream fos = null;
+        try
+        {
+            fos = new FileOutputStream( statsFile );
+            props.store( fos, "Last Scan Information, managed by Archiva. DO NOT EDIT" );
+            fos.flush();
+        }
+        finally
+        {
+            IOUtil.close( fos );
+        }
+    }
+
+    public void reset()
+    {
+        timestampStarted = 0;
+        timestampFinished = 0;
+        filesIncluded = 0;
+        filesConsumed = 0;
+        filesSkipped = 0;
+    }
+
+    public long getElapsedMilliseconds()
+    {
+        return timestampFinished - timestampStarted;
+    }
+
+    public long getFilesConsumed()
+    {
+        return filesConsumed;
+    }
+
+    public long getFilesIncluded()
+    {
+        return filesIncluded;
+    }
+
+    public ArtifactRepository getRepository()
+    {
+        return repository;
+    }
+
+    public long getTimestampFinished()
+    {
+        return timestampFinished;
+    }
+
+    public long getTimestampStarted()
+    {
+        return timestampStarted;
+    }
+
+    public long getFilesSkipped()
+    {
+        return filesSkipped;
+    }
+
+    public void setTimestampFinished( long timestampFinished )
+    {
+        this.timestampFinished = timestampFinished;
+    }
+
+    public void setTimestampStarted( long timestampStarted )
+    {
+        this.timestampStarted = timestampStarted;
+    }
+
+    public void dump( Logger logger )
+    {
+        logger.info( "----------------------------------------------------" );
+        logger.info( "Scan of Repository: " + repository.getId() );
+        logger.info( "   Started : " + toHumanTimestamp( this.getTimestampStarted() ) );
+        logger.info( "   Finished: " + toHumanTimestamp( this.getTimestampFinished() ) );
+        // TODO: pretty print ellapsed time.
+        logger.info( "   Duration: " + this.getElapsedMilliseconds() + "ms" );
+        logger.info( "   Files   : " + this.getFilesIncluded() );
+        logger.info( "   Consumed: " + this.getFilesConsumed() );
+        logger.info( "   Skipped : " + this.getFilesSkipped() );
+    }
+    
+    private String toHumanTimestamp( long timestamp )
+    {
+        SimpleDateFormat dateFormat = new SimpleDateFormat();
+        return dateFormat.format( new Date( timestamp ) );
+    }
+}

archiva-repository-layer/src/test/java/org/apache/maven/archiva/layer/AbstractRepositoryQueryLayerTestCase.java

@@ -45,7 +45,7 @@ public abstract class AbstractRepositoryQueryLayerTestCase
         throws Exception
     {
         super.setUp();
-        File repositoryDirectory = getTestFile( "src/test/repository" );
+        File repositoryDirectory = getTestFile( "src/test/repositories/repository" );
 
         artifactFactory = (ArtifactFactory) lookup( ArtifactFactory.ROLE );
         ArtifactRepositoryFactory factory = (ArtifactRepositoryFactory) lookup( ArtifactRepositoryFactory.ROLE );

archiva-repository-layer/src/test/repositories/legacy-repository/CVS/Root

@@ -0,0 +1 @@
+not a real CVS root - for testing exclusions

archiva-repository-layer/src/test/repositories/legacy-repository/org.apache.maven.update/jars/test-not-updated-1.0.jar

@@ -0,0 +1 @@
+dummy content. sample file only.

archiva-repository-layer/src/test/repositories/legacy-repository/org.apache.maven.update/jars/test-updated-1.0.jar

@@ -0,0 +1 @@
+dummy content. sample file only.