[MRM-1957] Spliting statistics plugin module

Adding new API module (metadata-statistics-api) for statistics and move the
jcr specific tasks to the metadata store module.

Providing interfaces for the statistics API.

Removing dependencies to store specific libraries (like JCR) from the statistic
modules.

3 files changed, 303 insertions, 2 deletions

diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
index 364930998..2b76db4af 100644
--- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
+++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
@@ -32,9 +32,12 @@ import org.apache.archiva.metadata.model.ProjectMetadata;
 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
 import org.apache.archiva.metadata.model.ProjectVersionReference;
 import org.apache.archiva.metadata.model.Scm;
+import org.apache.archiva.metadata.model.maven2.MavenArtifactFacet;
 import org.apache.archiva.metadata.repository.MetadataRepository;
 import org.apache.archiva.metadata.repository.MetadataRepositoryException;
 import org.apache.archiva.metadata.repository.MetadataResolutionException;
+import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics;
+import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsProvider;
 import org.apache.commons.lang.StringUtils;
 import org.apache.jackrabbit.commons.JcrUtils;
 import org.slf4j.Logger;
@@ -57,6 +60,7 @@ import javax.jcr.Workspace;
 import javax.jcr.nodetype.NodeTypeManager;
 import javax.jcr.nodetype.NodeTypeTemplate;
 import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
 import javax.jcr.query.QueryResult;
 import javax.jcr.query.Row;
 import javax.jcr.query.RowIterator;
@@ -80,7 +84,7 @@ import java.util.Set;
  * TODO revise reference storage
  */
 public class JcrMetadataRepository
-    implements MetadataRepository
+    implements MetadataRepository,RepositoryStatisticsProvider
 {
 
     private static final String JCR_LAST_MODIFIED = "jcr:lastModified";
@@ -113,7 +117,7 @@ public class JcrMetadataRepository
     }
 
 
-    static void initialize( Session session )
+    public static void initialize( Session session )
         throws RepositoryException
     {
 
@@ -1694,4 +1698,81 @@ public class JcrMetadataRepository
         }
         return this.jcrSession;
     }
+
+    @Override
+    public void populateStatistics( MetadataRepository repository, String repositoryId,
+                                            RepositoryStatistics repositoryStatistics )
+        throws MetadataRepositoryException
+    {
+        if (!(repository instanceof JcrMetadataRepository)) {
+            throw new MetadataRepositoryException( "The statistics population is only possible for JcrMetdataRepository implementations" );
+        }
+        Session session = (Session) repository.obtainAccess( Session.class );
+        // TODO: these may be best as running totals, maintained by observations on the properties in JCR
+
+        try
+        {
+            QueryManager queryManager = session.getWorkspace().getQueryManager();
+
+            // TODO: JCR-SQL2 query will not complete on a large repo in Jackrabbit 2.2.0 - see JCR-2835
+            //    Using the JCR-SQL2 variants gives
+            //      "org.apache.lucene.search.BooleanQuery$TooManyClauses: maxClauseCount is set to 1024"
+//            String whereClause = "WHERE ISDESCENDANTNODE([/repositories/" + repositoryId + "/content])";
+//            Query query = queryManager.createQuery( "SELECT size FROM [archiva:artifact] " + whereClause,
+//                                                    Query.JCR_SQL2 );
+            String whereClause = "WHERE jcr:path LIKE '/repositories/" + repositoryId + "/content/%'";
+            Query query = queryManager.createQuery( "SELECT size FROM archiva:artifact " + whereClause, Query.SQL );
+
+            QueryResult queryResult = query.execute();
+
+            Map<String, Integer> totalByType = new HashMap<>();
+            long totalSize = 0, totalArtifacts = 0;
+            for ( Row row : JcrUtils.getRows( queryResult ) )
+            {
+                Node n = row.getNode();
+                totalSize += row.getValue( "size" ).getLong();
+
+                String type;
+                if ( n.hasNode( MavenArtifactFacet.FACET_ID ) )
+                {
+                    Node facetNode = n.getNode( MavenArtifactFacet.FACET_ID );
+                    type = facetNode.getProperty( "type" ).getString();
+                }
+                else
+                {
+                    type = "Other";
+                }
+                Integer prev = totalByType.get( type );
+                totalByType.put( type, prev != null ? prev + 1 : 1 );
+
+                totalArtifacts++;
+            }
+
+            repositoryStatistics.setTotalArtifactCount( totalArtifacts );
+            repositoryStatistics.setTotalArtifactFileSize( totalSize );
+            for ( Map.Entry<String, Integer> entry : totalByType.entrySet() )
+            {
+                System.out.println("Setting count for type: "+entry.getKey()+" = "+entry.getValue());
+                repositoryStatistics.setTotalCountForType( entry.getKey(), entry.getValue() );
+            }
+
+            // The query ordering is a trick to ensure that the size is correct, otherwise due to lazy init it will be -1
+//            query = queryManager.createQuery( "SELECT * FROM [archiva:project] " + whereClause, Query.JCR_SQL2 );
+            query = queryManager.createQuery( "SELECT * FROM archiva:project " + whereClause + " ORDER BY jcr:score",
+                Query.SQL );
+            repositoryStatistics.setTotalProjectCount( query.execute().getRows().getSize() );
+
+//            query = queryManager.createQuery(
+//                "SELECT * FROM [archiva:namespace] " + whereClause + " AND namespace IS NOT NULL", Query.JCR_SQL2 );
+            query = queryManager.createQuery(
+                "SELECT * FROM archiva:namespace " + whereClause + " AND namespace IS NOT NULL ORDER BY jcr:score",
+                Query.SQL );
+            repositoryStatistics.setTotalGroupCount( query.execute().getRows().getSize() );
+        }
+        catch ( RepositoryException e )
+        {
+            throw new MetadataRepositoryException( e.getMessage(), e );
+        }
+    }
+
 }
diff --git a/archiva-modules/plugins/metadata-store-jcr/src/test/java/org/apache/archiva/metadata/repository/stats/JcrRepositoryStatisticsGatheringTest.java b/archiva-modules/plugins/metadata-store-jcr/src/test/java/org/apache/archiva/metadata/repository/stats/JcrRepositoryStatisticsGatheringTest.java
new file mode 100644
index 000000000..63bfffb6f
--- /dev/null
+++ b/archiva-modules/plugins/metadata-store-jcr/src/test/java/org/apache/archiva/metadata/repository/stats/JcrRepositoryStatisticsGatheringTest.java
@@ -0,0 +1,220 @@
+package org.apache.archiva.metadata.repository.stats;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import junit.framework.TestCase;
+import org.apache.archiva.metadata.model.MetadataFacetFactory;
+import org.apache.archiva.metadata.repository.AbstractMetadataRepositoryTest;
+import org.apache.archiva.metadata.repository.RepositorySessionFactory;
+import org.apache.archiva.metadata.repository.jcr.JcrMetadataRepository;
+import org.apache.archiva.metadata.repository.stats.model.DefaultRepositoryStatistics;
+import org.apache.archiva.test.utils.ArchivaBlockJUnit4ClassRunner;
+import org.apache.archiva.test.utils.ArchivaSpringJUnit4ClassRunner;
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.commons.JcrUtils;
+import org.apache.jackrabbit.core.TransientRepository;
+import org.apache.jackrabbit.core.config.RepositoryConfig;
+import org.apache.regexp.RE;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.context.ApplicationContext;
+import org.springframework.test.context.ContextConfiguration;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.jcr.ImportUUIDBehavior;
+import javax.jcr.NamespaceRegistry;
+import javax.jcr.Node;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Workspace;
+import javax.jcr.nodetype.NodeTypeManager;
+import javax.jcr.nodetype.NodeTypeTemplate;
+import java.io.File;
+import java.io.IOException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
+
+import static org.junit.Assert.assertEquals;
+
+@RunWith( ArchivaSpringJUnit4ClassRunner.class )
+@ContextConfiguration( locations = { "classpath*:/META-INF/spring-context.xml", "classpath*:/spring-context.xml" } )
+public class JcrRepositoryStatisticsGatheringTest extends TestCase
+
+{
+    private static final int TOTAL_FILE_COUNT = 1000;
+
+    private static final int NEW_FILE_COUNT = 500;
+
+    private static final String TEST_REPO = "test-repo";
+
+    JcrMetadataRepository repository;
+
+    @Inject
+    private RepositorySessionFactory repositorySessionFactory;
+
+    @Inject
+    private ApplicationContext applicationContext;
+
+    @Inject
+    @Named("repository")
+    Repository jcrRepository;
+
+    Session session;
+
+
+
+
+    @Before
+    public void setUp()
+        throws Exception
+    {
+
+        File directory = new File( "target/test-repositories" );
+        if ( directory.exists() )
+        {
+            FileUtils.deleteDirectory( directory );
+        }
+
+        Map<String, MetadataFacetFactory> factories = AbstractMetadataRepositoryTest.createTestMetadataFacetFactories();
+
+        assertNotNull( jcrRepository );
+        // TODO: probably don't need to use Spring for this
+        JcrMetadataRepository jcrMetadataRepository = new JcrMetadataRepository( factories, jcrRepository );
+
+        try
+        {
+            session = jcrMetadataRepository.getJcrSession();
+
+            // set up namespaces, etc.
+            JcrMetadataRepository.initialize( session );
+
+            // removing content is faster than deleting and re-copying the files from target/jcr
+            session.getRootNode().getNode( "repositories" ).remove();
+        }
+        catch ( RepositoryException e )
+        {
+            // ignore
+        }
+
+        this.repository = jcrMetadataRepository;
+    }
+
+    private static void registerMixinNodeType( NodeTypeManager nodeTypeManager, String type )
+        throws RepositoryException
+    {
+        NodeTypeTemplate nodeType = nodeTypeManager.createNodeTypeTemplate();
+        nodeType.setMixin( true );
+        nodeType.setName( type );
+        nodeTypeManager.registerNodeType( nodeType, false );
+    }
+
+    @After
+    public void tearDown()
+        throws Exception
+    {
+        if (repository!=null)
+        {
+            repository.close( );
+        }
+
+    }
+
+    @Test
+    public void testJcrStatisticsQuery()
+        throws Exception
+    {
+        Calendar cal = Calendar.getInstance();
+        Date endTime = cal.getTime();
+        cal.add( Calendar.HOUR, -1 );
+        Date startTime = cal.getTime();
+
+        loadContentIntoRepo( TEST_REPO );
+        loadContentIntoRepo( "another-repo" );
+
+        DefaultRepositoryStatistics testedStatistics = new DefaultRepositoryStatistics();
+        testedStatistics.setNewFileCount( NEW_FILE_COUNT );
+        testedStatistics.setTotalFileCount( TOTAL_FILE_COUNT );
+        testedStatistics.setScanStartTime( startTime );
+        testedStatistics.setScanEndTime( endTime );
+
+        repository.populateStatistics( repository, TEST_REPO, testedStatistics );
+
+        DefaultRepositoryStatistics expectedStatistics = new DefaultRepositoryStatistics();
+        expectedStatistics.setNewFileCount( NEW_FILE_COUNT );
+        expectedStatistics.setTotalFileCount( TOTAL_FILE_COUNT );
+        expectedStatistics.setScanEndTime( endTime );
+        expectedStatistics.setScanStartTime( startTime );
+        expectedStatistics.setTotalArtifactFileSize( 95954585 );
+        expectedStatistics.setTotalArtifactCount( 269 );
+        expectedStatistics.setTotalGroupCount( 1 );
+        expectedStatistics.setTotalProjectCount( 43 );
+        expectedStatistics.setTotalCountForType( "zip", 1 );
+        expectedStatistics.setTotalCountForType( "gz", 1 ); // FIXME: should be tar.gz
+        expectedStatistics.setTotalCountForType( "java-source", 10 );
+        expectedStatistics.setTotalCountForType( "jar", 108 );
+        expectedStatistics.setTotalCountForType( "xml", 3 );
+        expectedStatistics.setTotalCountForType( "war", 2 );
+        expectedStatistics.setTotalCountForType( "pom", 144 );
+        expectedStatistics.setRepositoryId( TEST_REPO );
+
+        System.out.println(testedStatistics.getTotalCountForType());
+
+        assertEquals( NEW_FILE_COUNT, testedStatistics.getNewFileCount());
+        assertEquals( TOTAL_FILE_COUNT, testedStatistics.getTotalFileCount() );
+        assertEquals( endTime, testedStatistics.getScanEndTime() );
+        assertEquals( startTime, testedStatistics.getScanStartTime() );
+        assertEquals( 95954585, testedStatistics.getTotalArtifactFileSize() );
+        assertEquals( 269, testedStatistics.getTotalArtifactCount() );
+        assertEquals( 1, testedStatistics.getTotalGroupCount() );
+        assertEquals( 43, testedStatistics.getTotalProjectCount() );
+        assertEquals( 1, testedStatistics.getTotalCountForType( "zip" ) );
+        assertEquals( 1, testedStatistics.getTotalCountForType( "gz" ) );
+        assertEquals( 10, testedStatistics.getTotalCountForType( "java-source" ) );
+        assertEquals( 108, testedStatistics.getTotalCountForType( "jar" ) );
+        assertEquals( 3, testedStatistics.getTotalCountForType( "xml" ) );
+        assertEquals( 2, testedStatistics.getTotalCountForType( "war" ) );
+        assertEquals( 144, testedStatistics.getTotalCountForType( "pom" ) );
+        assertEquals( 10, testedStatistics.getTotalCountForType( "java-source" ) );
+
+
+    }
+
+    private void loadContentIntoRepo( String repoId )
+        throws RepositoryException, IOException
+    {
+        Node n = JcrUtils.getOrAddNode( session.getRootNode(), "repositories" );
+        n = JcrUtils.getOrAddNode( n, repoId );
+        n = JcrUtils.getOrAddNode( n, "content" );
+        n = JcrUtils.getOrAddNode( n, "org" );
+        n = JcrUtils.getOrAddNode( n, "apache" );
+
+        GZIPInputStream inputStream = new GZIPInputStream( getClass().getResourceAsStream( "/artifacts.xml.gz" ) );
+        session.importXML( n.getPath(), inputStream, ImportUUIDBehavior.IMPORT_UUID_CREATE_NEW );
+        session.save();
+    }
+}
diff --git a/archiva-modules/plugins/metadata-store-jcr/src/test/resources/artifacts.xml.gz b/archiva-modules/plugins/metadata-store-jcr/src/test/resources/artifacts.xml.gz
new file mode 100644
index 000000000..0d8a41e3b
--- /dev/null
+++ b/archiva-modules/plugins/metadata-store-jcr/src/test/resources/artifacts.xml.gz