[MRM-1283] improve performance of on-demand resolution from storage. Still could be improved.

git-svn-id: https://svn.apache.org/repos/asf/archiva/branches/MRM-1025@885095 13f79535-47bb-0310-9956-ffa450edef68

archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/DefaultMetadataResolver.java

@@ -19,12 +19,15 @@ package org.apache.archiva.metadata.repository;
  * under the License.
  */
 
+import java.util.ArrayList;
 import java.util.Collection;
 
 import org.apache.archiva.metadata.model.Dependency;
 import org.apache.archiva.metadata.model.ProjectMetadata;
 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
 import org.apache.archiva.metadata.model.ProjectVersionReference;
+import org.apache.archiva.metadata.repository.filter.ExcludesFilter;
+import org.apache.archiva.metadata.repository.storage.StorageMetadataResolver;
 
 /**
  * @plexus.component role="org.apache.archiva.metadata.repository.MetadataResolver"
@@ -44,7 +47,7 @@ public class DefaultMetadataResolver
      *
      * @plexus.requirement role-hint="maven2"
      */
-    private MetadataResolver storageResolver;
+    private StorageMetadataResolver storageResolver;
 
     public ProjectMetadata getProject( String repoId, String namespace, String projectId )
     {
@@ -104,56 +107,79 @@ public class DefaultMetadataResolver
 
     public Collection<String> getRootNamespaces( String repoId )
     {
-        Collection<String> rootNamespaces = metadataRepository.getRootNamespaces( repoId );
-
-        // TODO: may want caching on this
-        Collection<String> storageRootNamespaces = storageResolver.getRootNamespaces( repoId );
-        if ( storageRootNamespaces != null && !storageRootNamespaces.equals( rootNamespaces ) )
+        Collection<String> namespaces = metadataRepository.getRootNamespaces( repoId );
+        Collection<String> storageNamespaces =
+            storageResolver.getRootNamespaces( repoId, new ExcludesFilter<String>( namespaces ) );
+        if ( storageNamespaces != null && !storageNamespaces.isEmpty() )
         {
-            // TODO: update the metadata repository
-            rootNamespaces = storageRootNamespaces;
+            for ( String n : storageNamespaces )
+            {
+                metadataRepository.updateNamespace( repoId, n );
+            }
+            namespaces = new ArrayList<String>( namespaces );
+            namespaces.addAll( storageNamespaces );
         }
-
-        return rootNamespaces;
+        return namespaces;
     }
 
     public Collection<String> getNamespaces( String repoId, String namespace )
     {
         Collection<String> namespaces = metadataRepository.getNamespaces( repoId, namespace );
-        // TODO: may want caching on this
-        Collection<String> storageNamespaces = storageResolver.getNamespaces( repoId, namespace );
-        if ( storageNamespaces != null && !storageNamespaces.equals( namespaces ) )
+        Collection<String> storageNamespaces =
+            storageResolver.getNamespaces( repoId, namespace, new ExcludesFilter<String>( namespaces ) );
+        if ( storageNamespaces != null && !storageNamespaces.isEmpty() )
         {
-            // TODO: update the metadata repository
-            namespaces = storageNamespaces;
+            for ( String n : storageNamespaces )
+            {
+                metadataRepository.updateNamespace( repoId, namespace + "." + n );
+            }
+            namespaces = new ArrayList<String>( namespaces );
+            namespaces.addAll( storageNamespaces );
         }
-
         return namespaces;
     }
 
     public Collection<String> getProjects( String repoId, String namespace )
     {
         Collection<String> projects = metadataRepository.getProjects( repoId, namespace );
-        // TODO: may want caching on this
-        Collection<String> storageProjects = storageResolver.getProjects( repoId, namespace );
-        if ( storageProjects != null && !storageProjects.equals( projects ) )
+        Collection<String> storageProjects =
+            storageResolver.getProjects( repoId, namespace, new ExcludesFilter<String>( projects ) );
+        if ( storageProjects != null && !storageProjects.isEmpty() )
         {
-            // TODO: update the metadata repository
-            projects = storageProjects;
+            for ( String projectId : storageProjects )
+            {
+                ProjectMetadata projectMetadata = storageResolver.getProject( repoId, namespace, projectId );
+                if ( projectMetadata != null )
+                {
+                    metadataRepository.updateProject( repoId, projectMetadata );
+                }
+            }
+            projects = new ArrayList<String>( projects );
+            projects.addAll( storageProjects );
         }
-
         return projects;
     }
 
     public Collection<String> getProjectVersions( String repoId, String namespace, String projectId )
+        throws MetadataResolverException
     {
         Collection<String> projectVersions = metadataRepository.getProjectVersions( repoId, namespace, projectId );
-        // TODO: may want caching on this
-        Collection<String> storageProjectVersions = storageResolver.getProjectVersions( repoId, namespace, projectId );
-        if ( storageProjectVersions != null && !storageProjectVersions.equals( projectVersions ) )
+        Collection<String> storageProjectVersions = storageResolver.getProjectVersions( repoId, namespace, projectId,
+                                                                                        new ExcludesFilter<String>(
+                                                                                            projectVersions ) );
+        if ( storageProjectVersions != null && !storageProjectVersions.isEmpty() )
         {
-            // TODO: update the metadata repository
-            projectVersions = storageProjectVersions;
+            for ( String projectVersion : storageProjectVersions )
+            {
+                ProjectVersionMetadata versionMetadata =
+                    storageResolver.getProjectVersion( repoId, namespace, projectId, projectVersion );
+                if ( versionMetadata != null )
+                {
+                    metadataRepository.updateProjectVersion( repoId, namespace, projectId, versionMetadata );
+                }
+            }
+            projectVersions = new ArrayList<String>( projectVersions );
+            projectVersions.addAll( storageProjectVersions );
         }
         return projectVersions;
     }

archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java

@@ -30,6 +30,7 @@ public interface MetadataRepository
     /**
      * Update metadata for a particular project in the metadata repository, or create it if it does not already exist.
      *
+     * @param repoId the repository the project is in
      * @param project the project metadata to create or update
      */
     void updateProject( String repoId, ProjectMetadata project );
@@ -42,4 +43,6 @@ public interface MetadataRepository
 
     void updateProjectReference( String repoId, String namespace, String projectId, String projectVersion,
                                  ProjectVersionReference reference );
+
+    void updateNamespace( String repoId, String namespace );
 }

archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataResolver.java

@@ -54,5 +54,6 @@ public interface MetadataResolver
 
     Collection<String> getProjects( String repoId, String namespace );
 
-    Collection<String> getProjectVersions( String repoId, String namespace, String projectId );
+    Collection<String> getProjectVersions( String repoId, String namespace, String projectId )
+        throws MetadataResolverException;
 }

archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/filter/AllFilter.java

@@ -0,0 +1,29 @@
+package org.apache.archiva.metadata.repository.filter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+public class AllFilter<T>
+    implements Filter<T>
+{
+    public boolean accept( T value )
+    {
+        return true;
+    }
+}

archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/filter/ExcludesFilter.java

@@ -0,0 +1,38 @@
+package org.apache.archiva.metadata.repository.filter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Collection;
+
+public class ExcludesFilter<T>
+    implements Filter<T>
+{
+    private Collection<T> excludes;
+
+    public ExcludesFilter( Collection<T> excludes )
+    {
+        this.excludes = excludes;
+    }
+
+    public boolean accept( T value )
+    {
+        return !excludes.contains( value );
+    }
+}

archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/filter/Filter.java

@@ -0,0 +1,25 @@
+package org.apache.archiva.metadata.repository.filter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+public interface Filter<T>
+{
+    boolean accept( T value );
+}

archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/storage/StorageMetadataResolver.java

@@ -0,0 +1,37 @@
+package org.apache.archiva.metadata.repository.storage;
+
+import java.util.Collection;
+
+import org.apache.archiva.metadata.repository.MetadataResolver;
+import org.apache.archiva.metadata.repository.filter.Filter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+public interface StorageMetadataResolver
+    extends MetadataResolver
+{
+    Collection<String> getRootNamespaces( String repoId, Filter<String> filter );
+
+    Collection<String> getNamespaces( String repoId, String namespace, Filter<String> filter );
+
+    Collection<String> getProjects( String repoId, String namespace, Filter<String> filter );
+
+    Collection<String> getProjectVersions( String repoId, String namespace, String projectId, Filter<String> filter );    
+}

archiva-modules/plugins/maven2-repository/src/main/java/org/apache/archiva/metadata/repository/storage/maven2/Maven2RepositoryMetadataResolver.java

@@ -30,9 +30,11 @@ import java.util.List;
 import org.apache.archiva.metadata.model.ProjectMetadata;
 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
 import org.apache.archiva.metadata.model.ProjectVersionReference;
-import org.apache.archiva.metadata.repository.MetadataResolver;
 import org.apache.archiva.metadata.repository.MetadataResolverException;
+import org.apache.archiva.metadata.repository.filter.AllFilter;
+import org.apache.archiva.metadata.repository.filter.Filter;
 import org.apache.archiva.metadata.repository.storage.RepositoryPathTranslator;
+import org.apache.archiva.metadata.repository.storage.StorageMetadataResolver;
 import org.apache.maven.archiva.common.utils.VersionUtil;
 import org.apache.maven.archiva.configuration.ArchivaConfiguration;
 import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
@@ -53,10 +55,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * @plexus.component role="org.apache.archiva.metadata.repository.MetadataResolver" role-hint="maven2"
+ * @plexus.component role="org.apache.archiva.metadata.repository.storage.StorageMetadataResolver" role-hint="maven2"
  */
 public class Maven2RepositoryMetadataResolver
-    implements MetadataResolver
+    implements StorageMetadataResolver
 {
     /**
      * @plexus.requirement
@@ -77,25 +79,12 @@ public class Maven2RepositoryMetadataResolver
 
     private static final String METADATA_FILENAME = "maven-metadata.xml";
 
-    private static final FilenameFilter DIRECTORY_FILTER = new FilenameFilter()
-    {
-        public boolean accept( File dir, String name )
-        {
-            if ( name.startsWith( "." ) )
-            {
-                return false;
-            }
-            else if ( !new File( dir, name ).isDirectory() )
-            {
-                return false;
-            }
-            return true;
-        }
-    };
+    private static final Filter<String> ALL = new AllFilter<String>();
 
     public ProjectMetadata getProject( String repoId, String namespace, String projectId )
     {
-        throw new UnsupportedOperationException();
+        // TODO: could natively implement the "shared model" concept from the browse action to avoid needing it there?
+        return null;
     }
 
     public ProjectVersionMetadata getProjectVersion( String repoId, String namespace, String projectId,
@@ -291,24 +280,30 @@ public class Maven2RepositoryMetadataResolver
         return ci;
     }
 
-    // TODO: evidence that storage and resolver != repository API - split the interface up
     public Collection<String> getArtifactVersions( String repoId, String namespace, String projectId,
                                                    String projectVersion )
     {
+        // TODO: useful, but not implemented yet, not called from DefaultMetadataResolver
         throw new UnsupportedOperationException();
     }
 
     public Collection<ProjectVersionReference> getProjectReferences( String repoId, String namespace, String projectId,
                                                                      String projectVersion )
     {
+        // Can't be determined on a Maven 2 repository
         throw new UnsupportedOperationException();
     }
 
     public Collection<String> getRootNamespaces( String repoId )
+    {
+        return getRootNamespaces( repoId, ALL );
+    }
+
+    public Collection<String> getRootNamespaces( String repoId, Filter<String> filter )
     {
         File dir = getRepositoryBasedir( repoId );
 
-        String[] files = dir.list( DIRECTORY_FILTER );
+        String[] files = dir.list( new DirectoryFilter( filter ) );
         return files != null ? Arrays.asList( files ) : Collections.<String>emptyList();
     }
 
@@ -321,17 +316,22 @@ public class Maven2RepositoryMetadataResolver
     }
 
     public Collection<String> getNamespaces( String repoId, String namespace )
+    {
+        return getNamespaces( repoId, namespace, ALL );
+    }
+
+    public Collection<String> getNamespaces( String repoId, String namespace, Filter<String> filter )
     {
         File dir = pathTranslator.toFile( getRepositoryBasedir( repoId ), namespace );
 
         // scan all the directories which are potential namespaces. Any directories known to be projects are excluded
         Collection<String> namespaces = new ArrayList<String>();
-        File[] files = dir.listFiles( DIRECTORY_FILTER );
+        File[] files = dir.listFiles( new DirectoryFilter( filter ) );
         if ( files != null )
         {
             for ( File file : files )
             {
-                if ( !isProject( file ) )
+                if ( !isProject( file, filter ) )
                 {
                     namespaces.add( file.getName() );
                 }
@@ -341,17 +341,22 @@ public class Maven2RepositoryMetadataResolver
     }
 
     public Collection<String> getProjects( String repoId, String namespace )
+    {
+        return getProjects( repoId, namespace, ALL );
+    }
+
+    public Collection<String> getProjects( String repoId, String namespace, Filter<String> filter )
     {
         File dir = pathTranslator.toFile( getRepositoryBasedir( repoId ), namespace );
 
         // scan all directories in the namespace, and only include those that are known to be projects
         Collection<String> projects = new ArrayList<String>();
-        File[] files = dir.listFiles( DIRECTORY_FILTER );
+        File[] files = dir.listFiles( new DirectoryFilter( filter ) );
         if ( files != null )
         {
             for ( File file : files )
             {
-                if ( isProject( file ) )
+                if ( isProject( file, filter ) )
                 {
                     projects.add( file.getName() );
                 }
@@ -361,27 +366,24 @@ public class Maven2RepositoryMetadataResolver
     }
 
     public Collection<String> getProjectVersions( String repoId, String namespace, String projectId )
+    {
+        return getProjectVersions( repoId, namespace, projectId, ALL );
+    }
+
+    public Collection<String> getProjectVersions( String repoId, String namespace, String projectId,
+                                                  Filter<String> filter )
     {
         File dir = pathTranslator.toFile( getRepositoryBasedir( repoId ), namespace, projectId );
 
         // all directories in a project directory can be considered a version
-        Collection<String> projectVersions = new ArrayList<String>();
-        String[] files = dir.list( DIRECTORY_FILTER );
+        String[] files = dir.list( new DirectoryFilter( filter ) );
         return files != null ? Arrays.asList( files ) : Collections.<String>emptyList();
     }
 
-    private boolean isProject( File dir )
+    private boolean isProject( File dir, Filter<String> filter )
     {
-        // if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
-        MavenRepositoryMetadata metadata = readMetadata( dir );
-        if ( metadata != null && dir.getName().equals( metadata.getArtifactId() ) )
-        {
-            return true;
-        }
-
-        // if metadata is missing, scan directories for a valid project version subdirectory, meaning this must be a
-        // project directory
-        File[] files = dir.listFiles( DIRECTORY_FILTER );
+        // scan directories for a valid project version subdirectory, meaning this must be a project directory
+        File[] files = dir.listFiles( new DirectoryFilter( filter ) );
         if ( files != null )
         {
             for ( File file : files )
@@ -392,6 +394,14 @@ public class Maven2RepositoryMetadataResolver
                 }
             }
         }
+
+        // if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
+        MavenRepositoryMetadata metadata = readMetadata( dir );
+        if ( metadata != null && dir.getName().equals( metadata.getArtifactId() ) )
+        {
+            return true;
+        }
+
         return false;
     }
 
@@ -400,14 +410,7 @@ public class Maven2RepositoryMetadataResolver
         final String artifactId = dir.getParentFile().getName();
         final String projectVersion = dir.getName();
 
-        // if a metadata file is present, check if this is the "version" directory, marking it as a project version
-        MavenRepositoryMetadata metadata = readMetadata( dir );
-        if ( metadata != null && projectVersion.equals( metadata.getVersion() ) )
-        {
-            return true;
-        }
-
-        // if metadata is missing, check if there is a POM artifact file to ensure it is a version directory
+        // check if there is a POM artifact file to ensure it is a version directory
         File[] files;
         if ( VersionUtil.isSnapshot( projectVersion ) )
         {
@@ -439,7 +442,19 @@ public class Maven2RepositoryMetadataResolver
                 }
             } );
         }
-        return files != null && files.length > 0;
+        if ( files != null && files.length > 0 )
+        {
+            return true;
+        }
+
+        // if a metadata file is present, check if this is the "version" directory, marking it as a project version
+        MavenRepositoryMetadata metadata = readMetadata( dir );
+        if ( metadata != null && projectVersion.equals( metadata.getVersion() ) )
+        {
+            return true;
+        }
+
+        return false;
     }
 
     private MavenRepositoryMetadata readMetadata( File directory )
@@ -459,4 +474,32 @@ public class Maven2RepositoryMetadataResolver
         }
         return metadata;
     }
+
+    private static class DirectoryFilter
+        implements FilenameFilter
+    {
+        private final Filter<String> filter;
+
+        public DirectoryFilter( Filter<String> filter )
+        {
+            this.filter = filter;
+        }
+
+        public boolean accept( File dir, String name )
+        {
+            if ( !filter.accept( name ) )
+            {
+                return false;
+            }
+            else if ( name.startsWith( "." ) )
+            {
+                return false;
+            }
+            else if ( !new File( dir, name ).isDirectory() )
+            {
+                return false;
+            }
+            return true;
+        }
+    }
 }

archiva-modules/plugins/maven2-repository/src/test/java/org/apache/archiva/metadata/repository/storage/maven2/Maven2RepositoryMetadataResolverTest.java

@@ -28,8 +28,8 @@ import org.apache.archiva.metadata.model.Dependency;
 import org.apache.archiva.metadata.model.License;
 import org.apache.archiva.metadata.model.MailingList;
 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
-import org.apache.archiva.metadata.repository.MetadataResolver;
 import org.apache.archiva.metadata.repository.MetadataResolverException;
+import org.apache.archiva.metadata.repository.storage.StorageMetadataResolver;
 import org.apache.maven.archiva.configuration.ArchivaConfiguration;
 import org.apache.maven.archiva.configuration.Configuration;
 import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
@@ -61,7 +61,7 @@ public class Maven2RepositoryMetadataResolverTest
         c.addManagedRepository( testRepo );
         configuration.save( c );
 
-        resolver = (Maven2RepositoryMetadataResolver) lookup( MetadataResolver.class, "maven2" );
+        resolver = (Maven2RepositoryMetadataResolver) lookup( StorageMetadataResolver.class, "maven2" );
     }
 
     public void testGetProjectVersionMetadata()

archiva-modules/plugins/metadata-repository-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java

@@ -86,13 +86,13 @@ public class FileMetadataRepository
     private void updateProject( String repoId, String namespace, String id )
     {
         // TODO: this is a more braindead implementation than we would normally expect, for prototyping purposes
+        updateNamespace( repoId, namespace );
+
         try
         {
             File namespaceDirectory = new File( this.directory, repoId + "/" + namespace );
             Properties properties = new Properties();
             properties.setProperty( "namespace", namespace );
-            writeProperties( properties, namespaceDirectory, NAMESPACE_METADATA_KEY );
-
             properties.setProperty( "id", id );
             writeProperties( properties, new File( namespaceDirectory, id ), PROJECT_METADATA_KEY );
 
@@ -217,6 +217,23 @@ public class FileMetadataRepository
         }
     }
 
+    public void updateNamespace( String repoId, String namespace )
+    {
+        try
+        {
+            File namespaceDirectory = new File( this.directory, repoId + "/" + namespace );
+            Properties properties = new Properties();
+            properties.setProperty( "namespace", namespace );
+            writeProperties( properties, namespaceDirectory, NAMESPACE_METADATA_KEY );
+
+        }
+        catch ( IOException e )
+        {
+            // TODO!
+            e.printStackTrace();
+        }
+    }
+
     private String join( Collection<String> ids )
     {
         if ( !ids.isEmpty() )