diff options
author | Martin Stockhammer <martin_s@apache.org> | 2017-07-16 00:13:40 +0200 |
---|---|---|
committer | Martin Stockhammer <martin_s@apache.org> | 2017-07-16 17:56:54 +0200 |
commit | f637c02c714952a88b999d00ecf78bb18975d0d1 (patch) | |
tree | e40b9e2d247f286f4f28bcb27682969a16a3df2e | |
parent | 42df217cd095a98ab935885a1c6e6e4f10a37954 (diff) | |
download | archiva-f637c02c714952a88b999d00ecf78bb18975d0d1.tar.gz archiva-f637c02c714952a88b999d00ecf78bb18975d0d1.zip |
[MRM-1957] Spliting statistics plugin module
Adding new API module (metadata-statistics-api) for statistics and move the
jcr specific tasks to the metadata store module.
Providing interfaces for the statistics API.
Removing dependencies to store specific libraries (like JCR) from the statistic
modules.
33 files changed, 875 insertions, 337 deletions
diff --git a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/pom.xml b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/pom.xml index 06fe2e4db..46e9ccfdb 100644 --- a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/pom.xml +++ b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/pom.xml @@ -55,6 +55,10 @@ </dependency> <dependency> <groupId>org.apache.archiva</groupId> + <artifactId>repository-statistics</artifactId> + </dependency> + <dependency> + <groupId>org.apache.archiva</groupId> <artifactId>metadata-model</artifactId> </dependency> <dependency> diff --git a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/managed/DefaultManagedRepositoryAdmin.java b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/managed/DefaultManagedRepositoryAdmin.java index f398bca26..7975d1f46 100644 --- a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/managed/DefaultManagedRepositoryAdmin.java +++ b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/managed/DefaultManagedRepositoryAdmin.java @@ -35,7 +35,7 @@ import org.apache.archiva.metadata.repository.MetadataRepository; import org.apache.archiva.metadata.repository.MetadataRepositoryException; import org.apache.archiva.metadata.repository.RepositorySession; import org.apache.archiva.metadata.repository.RepositorySessionFactory; -import org.apache.archiva.metadata.repository.stats.RepositoryStatisticsManager; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsManager; import org.apache.archiva.redback.components.cache.Cache; import org.apache.archiva.redback.components.taskqueue.TaskQueueException; import org.apache.archiva.redback.role.RoleManager; diff --git a/archiva-modules/archiva-karaf/archiva-features/src/main/resources/features.xml b/archiva-modules/archiva-karaf/archiva-features/src/main/resources/features.xml index 82d7cef40..dc5f9f759 100644 --- a/archiva-modules/archiva-karaf/archiva-features/src/main/resources/features.xml +++ b/archiva-modules/archiva-karaf/archiva-features/src/main/resources/features.xml @@ -89,6 +89,7 @@ <feature version="[3,4)">spring</feature> <bundle>mvn:org.apache.archiva/metadata-model/${project.version}</bundle> <bundle>mvn:org.apache.archiva/metadata-repository-api/${project.version}</bundle> + <bundle>mvn:org.apache.archiva/metadata-statistics-api/${project.version}</bundle> </feature> <feature name="archiva-audit" description="Archiva Audit module" version="${project.version}" resolver="(obr)"> @@ -206,8 +207,6 @@ <feature name="archiva-repository-statistics" description="Archiva Repository Statistics module" version="${project.version}" resolver="(obr)"> <feature version="${project.version}">archiva-repository-layer</feature> - <bundle dependency="true">mvn:javax.jcr/jcr/${javax.jcr.version}</bundle> - <bundle dependency="true">mvn:org.apache.jackrabbit/jackrabbit-jcr-commons/${jackrabbit.version}</bundle> <bundle>mvn:org.apache.archiva/repository-statistics/${project.version}</bundle> </feature> diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-repository-api/pom.xml b/archiva-modules/archiva-scheduler/archiva-scheduler-repository-api/pom.xml index 122542a6b..e728a8b30 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-repository-api/pom.xml +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-repository-api/pom.xml @@ -56,7 +56,7 @@ </dependency> <dependency> <groupId>org.apache.archiva</groupId> - <artifactId>repository-statistics</artifactId> + <artifactId>metadata-statistics-api</artifactId> </dependency> <dependency> <groupId>org.apache.archiva</groupId> diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/main/java/org/apache/archiva/scheduler/repository/ArchivaRepositoryScanningTaskExecutor.java b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/main/java/org/apache/archiva/scheduler/repository/ArchivaRepositoryScanningTaskExecutor.java index 3493168af..1ac363cd9 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/main/java/org/apache/archiva/scheduler/repository/ArchivaRepositoryScanningTaskExecutor.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/main/java/org/apache/archiva/scheduler/repository/ArchivaRepositoryScanningTaskExecutor.java @@ -26,8 +26,8 @@ import org.apache.archiva.metadata.repository.MetadataRepository; import org.apache.archiva.metadata.repository.MetadataRepositoryException; import org.apache.archiva.metadata.repository.RepositorySession; import org.apache.archiva.metadata.repository.RepositorySessionFactory; -import org.apache.archiva.metadata.repository.stats.RepositoryStatistics; -import org.apache.archiva.metadata.repository.stats.RepositoryStatisticsManager; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsManager; import org.apache.archiva.repository.scanner.RepositoryContentConsumers; import org.apache.archiva.repository.scanner.RepositoryScanStatistics; import org.apache.archiva.repository.scanner.RepositoryScanner; diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/main/java/org/apache/archiva/scheduler/repository/DefaultRepositoryArchivaTaskScheduler.java b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/main/java/org/apache/archiva/scheduler/repository/DefaultRepositoryArchivaTaskScheduler.java index 3012a40ec..672869b18 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/main/java/org/apache/archiva/scheduler/repository/DefaultRepositoryArchivaTaskScheduler.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/main/java/org/apache/archiva/scheduler/repository/DefaultRepositoryArchivaTaskScheduler.java @@ -28,7 +28,7 @@ import org.apache.archiva.metadata.repository.MetadataRepository; import org.apache.archiva.metadata.repository.MetadataRepositoryException; import org.apache.archiva.metadata.repository.RepositorySession; import org.apache.archiva.metadata.repository.RepositorySessionFactory; -import org.apache.archiva.metadata.repository.stats.RepositoryStatisticsManager; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsManager; import org.apache.archiva.redback.components.scheduler.CronExpressionValidator; import org.apache.archiva.redback.components.scheduler.Scheduler; import org.apache.archiva.redback.components.taskqueue.TaskQueue; diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/AbstractArchivaRepositoryScanningTaskExecutorTest.java b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/AbstractArchivaRepositoryScanningTaskExecutorTest.java index dcddb7153..de7bd459d 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/AbstractArchivaRepositoryScanningTaskExecutorTest.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/AbstractArchivaRepositoryScanningTaskExecutorTest.java @@ -23,15 +23,13 @@ import junit.framework.TestCase; import org.apache.archiva.configuration.ArchivaConfiguration; import org.apache.archiva.configuration.ManagedRepositoryConfiguration; import org.apache.archiva.metadata.repository.MetadataRepository; -import org.apache.archiva.metadata.repository.stats.RepositoryStatisticsManager; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsManager; import org.apache.archiva.mock.MockRepositorySessionFactory; import org.apache.archiva.redback.components.taskqueue.execution.TaskExecutor; import org.apache.archiva.test.utils.ArchivaSpringJUnit4ClassRunner; -import org.apache.commons.lang.StringUtils; import org.codehaus.plexus.util.FileUtils; import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -40,7 +38,6 @@ import javax.inject.Inject; import javax.inject.Named; import java.io.File; import java.util.Calendar; -import java.util.List; import static org.mockito.Mockito.mock; diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/ArchivaRepositoryScanningTaskExecutorPhase2Test.java b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/ArchivaRepositoryScanningTaskExecutorPhase2Test.java index 6631339bb..885c9a6a8 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/ArchivaRepositoryScanningTaskExecutorPhase2Test.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/ArchivaRepositoryScanningTaskExecutorPhase2Test.java @@ -20,7 +20,8 @@ package org.apache.archiva.scheduler.repository; */ import org.apache.archiva.metadata.repository.MetadataRepositoryException; -import org.apache.archiva.metadata.repository.stats.RepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.DefaultRepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics; import org.apache.archiva.model.ArtifactReference; import org.apache.archiva.scheduler.repository.model.RepositoryTask; import org.codehaus.plexus.util.FileUtils; @@ -225,7 +226,7 @@ public class ArchivaRepositoryScanningTaskExecutorPhase2Test throws MetadataRepositoryException { Date date = Calendar.getInstance().getTime(); - RepositoryStatistics stats = new RepositoryStatistics(); + DefaultRepositoryStatistics stats = new DefaultRepositoryStatistics(); stats.setScanStartTime( new Date( date.getTime() - 1234567 ) ); stats.setScanEndTime( date ); stats.setNewFileCount( 31 ); diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/TestRepositoryStatisticsManager.java b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/TestRepositoryStatisticsManager.java index d27e3506f..7f0512c84 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/TestRepositoryStatisticsManager.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/scheduler/repository/TestRepositoryStatisticsManager.java @@ -21,8 +21,9 @@ package org.apache.archiva.scheduler.repository; import org.apache.archiva.metadata.repository.MetadataRepository; import org.apache.archiva.metadata.repository.MetadataRepositoryException; -import org.apache.archiva.metadata.repository.stats.RepositoryStatistics; -import org.apache.archiva.metadata.repository.stats.RepositoryStatisticsManager; +import org.apache.archiva.metadata.repository.stats.model.DefaultRepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsManager; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -59,7 +60,7 @@ public class TestRepositoryStatisticsManager { List<RepositoryStatistics> stats = getStatsList( repositoryId ); - RepositoryStatistics repositoryStatistics = new RepositoryStatistics(); + DefaultRepositoryStatistics repositoryStatistics = new DefaultRepositoryStatistics(); repositoryStatistics.setScanStartTime( startTime ); repositoryStatistics.setScanEndTime( endTime ); repositoryStatistics.setNewFileCount( newFiles ); diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/ArchivaRepositoryStatistics.java b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/ArchivaRepositoryStatistics.java index 8bf3ddccf..a1e5352cb 100644 --- a/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/ArchivaRepositoryStatistics.java +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/ArchivaRepositoryStatistics.java @@ -21,6 +21,7 @@ package org.apache.archiva.rest.api.model; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; import java.util.Date; +import java.util.Map; /** * @author Olivier Lamy @@ -50,6 +51,10 @@ public class ArchivaRepositoryStatistics private String lastScanDate; + private Map<String, Long> totalCountForType; + + private Map<String, Long> customValues; + public ArchivaRepositoryStatistics() { // no op @@ -155,6 +160,24 @@ public class ArchivaRepositoryStatistics this.lastScanDate = lastScanDate; } + public void setTotalCountForType(Map<String, Long> totalCountForType) { + this.totalCountForType = totalCountForType; + } + + public Map<String, Long> getTotalCountForType() { + return this.totalCountForType; + } + + public void setCustomValues(Map<String,Long> customValues) { + this.customValues = customValues; + } + + public Map<String,Long> getCustomValues() { + return this.customValues; + } + + + @Override public String toString() { @@ -170,7 +193,16 @@ public class ArchivaRepositoryStatistics sb.append( ", newFileCount=" ).append( newFileCount ); sb.append( ", duration=" ).append( duration ); sb.append( ", lastScanDate='" ).append( lastScanDate ).append( '\'' ); + addMapString( sb, totalCountForType ); + addMapString( sb, customValues ); sb.append( '}' ); return sb.toString(); } + + private void addMapString(StringBuilder builder, Map<String, Long> map) { + if (map!=null) + { + map.entrySet( ).stream( ).forEach( entry -> builder.append( ", " ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) ) ); + } + } } diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/services/ReportRepositoriesService.java b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/services/ReportRepositoriesService.java index 1bb718554..5f3eccf34 100644 --- a/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/services/ReportRepositoriesService.java +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/services/ReportRepositoriesService.java @@ -19,7 +19,7 @@ package org.apache.archiva.rest.api.services; */ import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet; -import org.apache.archiva.metadata.repository.stats.RepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics; import org.apache.archiva.redback.authorization.RedbackAuthorization; import org.apache.archiva.security.common.ArchivaRoleConstants; diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultManagedRepositoriesService.java b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultManagedRepositoriesService.java index 9c10c9f09..ed3ba676d 100644 --- a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultManagedRepositoriesService.java +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultManagedRepositoriesService.java @@ -24,8 +24,8 @@ import org.apache.archiva.admin.model.beans.ManagedRepository; import org.apache.archiva.metadata.repository.MetadataRepository; import org.apache.archiva.metadata.repository.MetadataRepositoryException; import org.apache.archiva.metadata.repository.RepositorySession; -import org.apache.archiva.metadata.repository.stats.RepositoryStatistics; -import org.apache.archiva.metadata.repository.stats.RepositoryStatisticsManager; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsManager; import org.apache.archiva.rest.api.model.ArchivaRepositoryStatistics; import org.apache.archiva.rest.api.services.ArchivaRestServiceException; import org.apache.archiva.rest.api.services.ManagedRepositoriesService; diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultReportRepositoriesService.java b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultReportRepositoriesService.java index b8c1c74a5..b968e7a85 100644 --- a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultReportRepositoriesService.java +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultReportRepositoriesService.java @@ -22,8 +22,9 @@ import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet; import org.apache.archiva.metadata.repository.MetadataRepository; import org.apache.archiva.metadata.repository.MetadataRepositoryException; import org.apache.archiva.metadata.repository.RepositorySession; -import org.apache.archiva.metadata.repository.stats.RepositoryStatistics; -import org.apache.archiva.metadata.repository.stats.RepositoryStatisticsManager; +import org.apache.archiva.metadata.repository.stats.model.DefaultRepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsManager; import org.apache.archiva.rest.api.services.ArchivaRestServiceException; import org.apache.archiva.rest.api.services.ReportRepositoriesService; import org.apache.commons.lang.StringUtils; diff --git a/archiva-modules/archiva-web/archiva-webdav/pom.xml b/archiva-modules/archiva-web/archiva-webdav/pom.xml index 2b08b3f29..33c876011 100644 --- a/archiva-modules/archiva-web/archiva-webdav/pom.xml +++ b/archiva-modules/archiva-web/archiva-webdav/pom.xml @@ -105,6 +105,10 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.jackrabbit</groupId> + <artifactId>jackrabbit-jcr-commons</artifactId> + </dependency> + <dependency> <groupId>joda-time</groupId> <artifactId>joda-time</artifactId> </dependency> diff --git a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavLocatorFactory.java b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavLocatorFactory.java index 9d23b7733..4bda01924 100644 --- a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavLocatorFactory.java +++ b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavLocatorFactory.java @@ -19,10 +19,10 @@ package org.apache.archiva.webdav; * under the License. */ +import org.apache.archiva.webdav.util.RepositoryPathUtil; import org.apache.jackrabbit.util.Text; import org.apache.jackrabbit.webdav.DavLocatorFactory; import org.apache.jackrabbit.webdav.DavResourceLocator; -import org.apache.archiva.webdav.util.RepositoryPathUtil; /** */ diff --git a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java index 75c961fbc..fef84df1b 100644 --- a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java +++ b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java @@ -101,7 +101,7 @@ public abstract class AbstractMetadataRepositoryTest protected Logger log = LoggerFactory.getLogger( getClass() ); - protected static Map<String, MetadataFacetFactory> createTestMetadataFacetFactories() + public static Map<String, MetadataFacetFactory> createTestMetadataFacetFactories() { Map<String, MetadataFacetFactory> factories = new HashMap<>(); factories.put( TEST_FACET_ID, new MetadataFacetFactory() diff --git a/archiva-modules/metadata/metadata-statistics-api/pom.xml b/archiva-modules/metadata/metadata-statistics-api/pom.xml new file mode 100644 index 000000000..251f2a729 --- /dev/null +++ b/archiva-modules/metadata/metadata-statistics-api/pom.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>metadata</artifactId> + <groupId>org.apache.archiva</groupId> + <version>3.0.0-SNAPSHOT</version> + </parent> + <artifactId>metadata-statistics-api</artifactId> + <packaging>bundle</packaging> + <name>Archiva Metadata :: Statistics API</name> + <dependencies> + <dependency> + <groupId>org.apache.archiva</groupId> + <artifactId>metadata-repository-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.archiva</groupId> + <artifactId>metadata-model-maven2</artifactId> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <configuration> + <instructions> + <Bundle-SymbolicName>org.apache.archiva.metadata.repository.api</Bundle-SymbolicName> + <Bundle-Version>${project.version}</Bundle-Version> + <Export-Package> + org.apache.archiva.metadata.repository.stats.model;version=${project.version} + </Export-Package> + <Import-Package> + org.apache.archiva.metadata.model;version=${project.version}, + org.apache.archiva.maven2.model;version=${project.version}, + org.slf4j;resolution:=optional, + </Import-Package> + </instructions> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/RepositoryStatistics.java b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/DefaultRepositoryStatistics.java index 71cee371e..0c8885161 100644 --- a/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/RepositoryStatistics.java +++ b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/DefaultRepositoryStatistics.java @@ -1,4 +1,4 @@ -package org.apache.archiva.metadata.repository.stats; +package org.apache.archiva.metadata.repository.stats.model; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -19,16 +19,18 @@ package org.apache.archiva.metadata.repository.stats; * under the License. */ -import org.apache.archiva.metadata.model.MetadataFacet; - import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.TimeZone; +import java.util.stream.Collectors; -public class RepositoryStatistics - implements MetadataFacet +/** + * Default statistics implementation + */ +public class DefaultRepositoryStatistics + implements RepositoryStatistics { private Date scanEndTime; @@ -46,17 +48,21 @@ public class RepositoryStatistics private long newFileCount; - public static String FACET_ID = "org.apache.archiva.metadata.repository.stats"; + public static final String SCAN_TIMESTAMP_FORMAT = "yyyy/MM/dd/HHmmss.SSS"; - static final String SCAN_TIMESTAMP_FORMAT = "yyyy/MM/dd/HHmmss.SSS"; - - private Map<String, Long> totalCountForType = new ZeroForNullHashMap<String, Long>(); + private Map<String, Long> totalCountForType = new ZeroForNullHashMap<>(); private static final TimeZone UTC_TIME_ZONE = TimeZone.getTimeZone( "UTC" ); private String repositoryId; - public Date getScanEndTime() + private Map<String, Long> customValues; + + public static final String TYPE_PREFIX = "count-type-"; + public static final String CUSTOM_PREFIX = "count-custom-"; + + @Override + public Date getScanEndTime( ) { return scanEndTime; } @@ -66,7 +72,8 @@ public class RepositoryStatistics this.scanEndTime = scanEndTime; } - public Date getScanStartTime() + @Override + public Date getScanStartTime( ) { return scanStartTime; } @@ -76,72 +83,86 @@ public class RepositoryStatistics this.scanStartTime = scanStartTime; } - public long getTotalArtifactCount() + @Override + public long getTotalArtifactCount( ) { return totalArtifactCount; } + @Override public void setTotalArtifactCount( long totalArtifactCount ) { this.totalArtifactCount = totalArtifactCount; } - public long getTotalArtifactFileSize() + @Override + public long getTotalArtifactFileSize( ) { return totalArtifactFileSize; } + @Override public void setTotalArtifactFileSize( long totalArtifactFileSize ) { this.totalArtifactFileSize = totalArtifactFileSize; } - public long getTotalFileCount() + @Override + public long getTotalFileCount( ) { return totalFileCount; } + @Override public void setTotalFileCount( long totalFileCount ) { this.totalFileCount = totalFileCount; } - public long getTotalGroupCount() + @Override + public long getTotalGroupCount( ) { return totalGroupCount; } + @Override public void setTotalGroupCount( long totalGroupCount ) { this.totalGroupCount = totalGroupCount; } - public long getTotalProjectCount() + @Override + public long getTotalProjectCount( ) { return totalProjectCount; } + @Override public void setTotalProjectCount( long totalProjectCount ) { this.totalProjectCount = totalProjectCount; } + @Override public void setNewFileCount( long newFileCount ) { this.newFileCount = newFileCount; } - public long getNewFileCount() + @Override + public long getNewFileCount( ) { return newFileCount; } - public long getDuration() + @Override + public long getDuration( ) { return scanEndTime.getTime() - scanStartTime.getTime(); } - public String getRepositoryId() + @Override + public String getRepositoryId( ) { return repositoryId; } @@ -174,8 +195,18 @@ public class RepositoryStatistics public Map<String, String> toProperties() { Map<String, String> properties = new HashMap<>(); - properties.put( "scanEndTime", String.valueOf( scanEndTime.getTime() ) ); - properties.put( "scanStartTime", String.valueOf( scanStartTime.getTime() ) ); + if (scanEndTime==null) { + properties.put("scanEndTime", "0"); + } else + { + properties.put( "scanEndTime", String.valueOf( scanEndTime.getTime( ) ) ); + } + if (scanStartTime==null) { + properties.put("scanStartTime","0"); + } else + { + properties.put( "scanStartTime", String.valueOf( scanStartTime.getTime( ) ) ); + } properties.put( "totalArtifactCount", String.valueOf( totalArtifactCount ) ); properties.put( "totalArtifactFileSize", String.valueOf( totalArtifactFileSize ) ); properties.put( "totalFileCount", String.valueOf( totalFileCount ) ); @@ -185,7 +216,12 @@ public class RepositoryStatistics properties.put( "repositoryId", repositoryId ); for ( Map.Entry<String, Long> entry : totalCountForType.entrySet() ) { - properties.put( "count-" + entry.getKey(), String.valueOf( entry.getValue() ) ); + properties.put( TYPE_PREFIX + entry.getKey(), String.valueOf( entry.getValue() ) ); + } + if (customValues!=null) { + for (Map.Entry<String, Long> entry : customValues.entrySet()) { + properties.put(CUSTOM_PREFIX+entry.getKey(), String.valueOf(entry.getValue())); + } } return properties; } @@ -205,13 +241,19 @@ public class RepositoryStatistics totalCountForType.clear(); for ( Map.Entry<String, String> entry : properties.entrySet() ) { - if ( entry.getKey().startsWith( "count-" ) ) + if ( entry.getKey().startsWith( TYPE_PREFIX ) ) { - totalCountForType.put( entry.getKey().substring( 6 ), Long.valueOf( entry.getValue() ) ); + totalCountForType.put( entry.getKey().substring( TYPE_PREFIX.length() ), Long.valueOf( entry.getValue() ) ); + } else if (entry.getKey().startsWith( CUSTOM_PREFIX )) { + if (customValues==null) { + createCustomValueMap(); + } + customValues.put(entry.getKey().substring( CUSTOM_PREFIX.length() ), Long.valueOf(entry.getValue())); } } } + @Override public boolean equals( Object o ) { @@ -224,7 +266,7 @@ public class RepositoryStatistics return false; } - RepositoryStatistics that = (RepositoryStatistics) o; + DefaultRepositoryStatistics that = (DefaultRepositoryStatistics) o; if ( newFileCount != that.newFileCount ) { @@ -262,12 +304,16 @@ public class RepositoryStatistics { return false; } - if ( !repositoryId.equals( that.repositoryId ) ) - { + if ( customValues==null && that.customValues!=null) { return false; } - - return true; + if ( customValues!=null && that.customValues==null) { + return false; + } + if (customValues!=null && !customValues.equals(that.customValues)) { + return false; + } + return repositoryId.equals( that.repositoryId ); } @Override @@ -283,6 +329,8 @@ public class RepositoryStatistics result = 31 * result + (int) ( newFileCount ^ ( newFileCount >>> 32 ) ); result = 31 * result + totalCountForType.hashCode(); result = 31 * result + repositoryId.hashCode(); + if (customValues!=null) + result = 31 * result + customValues.hashCode(); return result; } @@ -293,31 +341,72 @@ public class RepositoryStatistics ", totalArtifactCount=" + totalArtifactCount + ", totalArtifactFileSize=" + totalArtifactFileSize + ", totalFileCount=" + totalFileCount + ", totalGroupCount=" + totalGroupCount + ", totalProjectCount=" + totalProjectCount + ", newFileCount=" + newFileCount + ", totalCountForType=" + totalCountForType + ", " + - "repositoryId=" + repositoryId + '}'; + "repositoryId=" + repositoryId + + getCustomValueString() + + '}'; + } + + private String getCustomValueString() { + if (customValues==null) { + return ""; + } else { + return customValues.entrySet().stream().map(entry -> entry.getKey()+"="+entry.getValue()).collect( + Collectors.joining( ",") + ); + } } - public Map<String, Long> getTotalCountForType() + @Override + public Map<String, Long> getTotalCountForType( ) { return totalCountForType; } + @Override public long getTotalCountForType( String type ) { return totalCountForType.get( type ); } + @Override public void setTotalCountForType( String type, long count ) { - totalCountForType.put( type.replaceAll( "-", "_" ).replaceAll( "\\.", "_" ), count ); + totalCountForType.put( type, count ); } - - private static final class ZeroForNullHashMap<K, V extends Long> extends HashMap<K, V> + + @Override + public long getCustomValue( String fieldName ) + { + // Lazy evaluation, because it may not be used very often. + if (customValues==null) { + createCustomValueMap(); + } + return customValues.get(fieldName); + } + + @Override + public void setCustomValue( String fieldName, long count ) + { + // Lazy evaluation, because it may not be used very often. + if (customValues==null) { + createCustomValueMap(); + } + customValues.put(fieldName, count); + } + + private void createCustomValueMap( ) + { + customValues = new ZeroForNullHashMap<>(); + } + + + private static final class ZeroForNullHashMap<K> extends HashMap<K, Long> { @Override - public V get(Object key) { - V value = super.get( key ); + public Long get(Object key) { + Long value = super.get( key ); - return value != null ? value : ( V ) Long.valueOf( 0L ); + return ( value != null ) ? value : Long.valueOf( 0L ); } } } diff --git a/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryStatistics.java b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryStatistics.java new file mode 100644 index 000000000..e5fa94a9b --- /dev/null +++ b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryStatistics.java @@ -0,0 +1,109 @@ +package org.apache.archiva.metadata.repository.stats.model; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.metadata.model.MetadataFacet; + +import java.util.Date; +import java.util.Map; + +/** + * + * Provides statistics data of metadata repositories. + * + * @since 2.3 + */ +public interface RepositoryStatistics extends MetadataFacet +{ + String FACET_ID = "org.apache.archiva.metadata.repository.stats"; + + String getRepositoryId( ); + + Date getScanEndTime( ); + + Date getScanStartTime( ); + + long getTotalArtifactCount( ); + + void setTotalArtifactCount( long totalArtifactCount ); + + long getTotalArtifactFileSize( ); + + void setTotalArtifactFileSize( long totalArtifactFileSize ); + + long getTotalFileCount( ); + + void setTotalFileCount( long totalFileCount ); + + long getTotalGroupCount( ); + + void setTotalGroupCount( long totalGroupCount ); + + long getTotalProjectCount( ); + + void setTotalProjectCount( long totalProjectCount ); + + void setNewFileCount( long newFileCount ); + + long getNewFileCount( ); + + long getDuration( ); + + /** + * Statistics data by artifact type. + * + * @return A list of data keys and values + */ + Map<String, Long> getTotalCountForType( ); + + /** + * Returns the value for the given artifact type. + * + * @param type The artifact type + * @return The count value. + */ + long getTotalCountForType( String type ); + + /** + * Sets the value for the given artifact type. + * @param type The artifact type. + * @param count The count value. + */ + void setTotalCountForType( String type, long count ); + + /** + * Reads custom statistic values that are store implementation + * specific. + * + * @param fieldName A unique field name. + */ + long getCustomValue(String fieldName); + + /** + * Saves custom statistic values that are store implementation + * specific. The field name should be unique (e.g. prefixed by the + * package name of the data provider). + * + * @param fieldName A unique field name. + * @param count The statistic counter value + */ + void setCustomValue(String fieldName, long count); + +} diff --git a/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsManager.java b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryStatisticsManager.java index 18332647a..f47e8cc16 100644 --- a/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsManager.java +++ b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryStatisticsManager.java @@ -1,4 +1,4 @@ -package org.apache.archiva.metadata.repository.stats; +package org.apache.archiva.metadata.repository.stats.model; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -25,6 +25,12 @@ import org.apache.archiva.metadata.repository.MetadataRepositoryException; import java.util.Date; import java.util.List; +/** + * The statistics manager is the central entry point for handling repository + * statistics data. + * + * + */ public interface RepositoryStatisticsManager { RepositoryStatistics getLastStatistics( MetadataRepository metadataRepository, String repositoryId ) diff --git a/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryStatisticsProvider.java b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryStatisticsProvider.java new file mode 100644 index 000000000..112f86a11 --- /dev/null +++ b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryStatisticsProvider.java @@ -0,0 +1,47 @@ +package org.apache.archiva.metadata.repository.stats.model; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.metadata.repository.MetadataRepository; +import org.apache.archiva.metadata.repository.MetadataRepositoryException; + +/** + * + * This interface is used for populating statistics data. It should be implemented + * by metadata store implementations in the MetadataRepository class, if the store + * implementation can provide a faster implementation than walking the tree. + * + * @author Martin Stockhammer + * @since 2.3 + */ +public interface RepositoryStatisticsProvider +{ + + /** + * Populate the statistics object with the statistics data of this repository. + * + * @param repository The current metadata repository implementation + * @param repositoryId The repository Id + * @param statistics The statistics object that should be filled. + * @throws MetadataRepositoryException Is thrown, if an error occurs while accessing the repository + */ + void populateStatistics( MetadataRepository repository, String repositoryId, RepositoryStatistics statistics) + throws MetadataRepositoryException; +} diff --git a/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryWalkingStatisticsProvider.java b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryWalkingStatisticsProvider.java new file mode 100644 index 000000000..9cc426c8c --- /dev/null +++ b/archiva-modules/metadata/metadata-statistics-api/src/main/java/org/apache/archiva/metadata/repository/stats/model/RepositoryWalkingStatisticsProvider.java @@ -0,0 +1,106 @@ +package org.apache.archiva.metadata.repository.stats.model; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.metadata.model.ArtifactMetadata; +import org.apache.archiva.metadata.model.maven2.MavenArtifactFacet; +import org.apache.archiva.metadata.repository.MetadataRepository; +import org.apache.archiva.metadata.repository.MetadataRepositoryException; +import org.apache.archiva.metadata.repository.MetadataResolutionException; + +import java.util.Collection; + +/** + * + * This is a default implementation of a statistics provider that walks the tree and + * counts the artifacts found during the walk. + * The implementation is not very fast. If metadata store provider can improve the + * process by using store specific techniques (like query language) they should provide + * their own implementation. + * + * @author Martin Stockhammer + */ +public class RepositoryWalkingStatisticsProvider implements RepositoryStatisticsProvider +{ + + /** + * Walks each namespace of the given repository id and counts the artifacts. + * + * @param metadataRepository The repository implementation + * @param repositoryId The repository Id + * @param repositoryStatistics The statistics object that must be populated + * @throws MetadataRepositoryException Throws the repository exception, if an error occurs while accessing the repository. + */ + @Override + public void populateStatistics( MetadataRepository metadataRepository, String repositoryId, + RepositoryStatistics repositoryStatistics ) + throws MetadataRepositoryException + { + try + { + for ( String ns : metadataRepository.getRootNamespaces( repositoryId ) ) + { + walkRepository( metadataRepository, repositoryStatistics, repositoryId, ns ); + } + } + catch ( MetadataResolutionException e ) + { + throw new MetadataRepositoryException( e.getMessage(), e ); + } + } + + private void walkRepository( MetadataRepository metadataRepository, RepositoryStatistics stats, String repositoryId, + String ns ) + throws MetadataResolutionException + { + for ( String namespace : metadataRepository.getNamespaces( repositoryId, ns ) ) + { + walkRepository( metadataRepository, stats, repositoryId, ns + "." + namespace ); + } + + Collection<String> projects = metadataRepository.getProjects( repositoryId, ns ); + if ( !projects.isEmpty() ) + { + stats.setTotalGroupCount( stats.getTotalGroupCount() + 1 ); + stats.setTotalProjectCount( stats.getTotalProjectCount() + projects.size() ); + + for ( String project : projects ) + { + for ( String version : metadataRepository.getProjectVersions( repositoryId, ns, project ) ) + { + for ( ArtifactMetadata artifact : metadataRepository.getArtifacts( repositoryId, ns, project, + version ) ) + { + stats.setTotalArtifactCount( stats.getTotalArtifactCount() + 1 ); + stats.setTotalArtifactFileSize( stats.getTotalArtifactFileSize() + artifact.getSize() ); + + MavenArtifactFacet facet = + (MavenArtifactFacet) artifact.getFacet( MavenArtifactFacet.FACET_ID ); + if ( facet != null ) + { + String type = facet.getType(); + stats.setTotalCountForType( type, stats.getTotalCountForType( type ) + 1 ); + } + } + } + } + } + } +} diff --git a/archiva-modules/metadata/metadata-statistics-api/src/test/java/org/apache/archiva/metadata/repository/stats/model/DefaultRepositoryStatisticsTest.java b/archiva-modules/metadata/metadata-statistics-api/src/test/java/org/apache/archiva/metadata/repository/stats/model/DefaultRepositoryStatisticsTest.java new file mode 100644 index 000000000..eb38a3954 --- /dev/null +++ b/archiva-modules/metadata/metadata-statistics-api/src/test/java/org/apache/archiva/metadata/repository/stats/model/DefaultRepositoryStatisticsTest.java @@ -0,0 +1,109 @@ +package org.apache.archiva.metadata.repository.stats.model; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +/** + * @author Martin Stockhammer <martin_s@apache.org> + */ +public class DefaultRepositoryStatisticsTest +{ + @Test + public void toProperties( ) throws Exception + { + DefaultRepositoryStatistics stats = new DefaultRepositoryStatistics(); + Date startTime = new Date(); + Date endTime = new Date(); + stats.setScanStartTime( startTime ); + stats.setScanEndTime( endTime ); + stats.setTotalFileCount( 500 ); + stats.setNewFileCount( 10 ); + stats.setRepositoryId( "test-repo" ); + stats.setTotalArtifactCount( 300 ); + stats.setTotalArtifactFileSize( 4848484 ); + stats.setTotalGroupCount( 4 ); + stats.setTotalProjectCount( 6 ); + stats.setCustomValue( "test.value.1", 55 ); + stats.setCustomValue( "test.value.2", 44); + stats.setTotalCountForType( "java-source",13 ); + stats.setTotalCountForType( "pom", 5 ); + + Map<String, String> props = stats.toProperties( ); + + assertEquals( "500", props.get("totalFileCount") ); + assertEquals( "10", props.get("newFileCount")); + assertEquals( "300", props.get("totalArtifactCount")); + assertEquals( "4848484", props.get("totalArtifactFileSize")); + assertEquals("4", props.get("totalGroupCount")); + assertEquals("6", props.get("totalProjectCount")); + assertEquals("55",props.get("count-custom-test.value.1" )); + assertEquals("44", props.get("count-custom-test.value.2")); + assertEquals("13", props.get("count-type-java-source")); + assertEquals("5", props.get("count-type-pom")); + assertEquals( String.valueOf(startTime.getTime()), props.get("scanStartTime")); + assertEquals( String.valueOf(endTime.getTime()), props.get("scanEndTime")); + } + + @Test + public void fromProperties( ) throws Exception + { + DefaultRepositoryStatistics stats = new DefaultRepositoryStatistics( ); + Date startTime = new Date(); + Date endTime = new Date(); + Map<String,String> props = new HashMap<>( ); + props.put("totalFileCount","501"); + props.put("newFileCount","11"); + props.put("totalArtifactCount","301"); + props.put("totalArtifactFileSize","473565557"); + props.put("totalGroupCount","5"); + props.put("totalProjectCount","7"); + props.put("count-custom-test.value.1","56"); + props.put("count-custom-test.value.2","45"); + props.put("count-type-java-source","14"); + props.put("count-type-pom","6"); + props.put("scanStartTime", String.valueOf(startTime.getTime())); + props.put("scanEndTime", String.valueOf(endTime.getTime())); + + stats.fromProperties( props ); + + assertEquals(501,stats.getTotalFileCount()); + assertEquals(11,stats.getNewFileCount()); + assertEquals(301, stats.getTotalArtifactCount()); + assertEquals(473565557, stats.getTotalArtifactFileSize()); + assertEquals(5, stats.getTotalGroupCount()); + assertEquals(7, stats.getTotalProjectCount()); + assertEquals(56, stats.getCustomValue( "test.value.1" )); + assertEquals(45, stats.getCustomValue( "test.value.2" )); + assertEquals(14, stats.getTotalCountForType( "java-source" )); + assertEquals(6, stats.getTotalCountForType( "pom" )); + assertEquals(startTime, stats.getScanStartTime()); + assertEquals( endTime, stats.getScanEndTime() ); + } + + + +}
\ No newline at end of file diff --git a/archiva-modules/metadata/pom.xml b/archiva-modules/metadata/pom.xml index 5ce399dea..1c1ba6978 100644 --- a/archiva-modules/metadata/pom.xml +++ b/archiva-modules/metadata/pom.xml @@ -31,6 +31,7 @@ <module>metadata-model</module> <module>metadata-repository-api</module> <module>metadata-model-maven2</module> + <module>metadata-statistics-api</module> <module>test-repository</module> </modules> </project>
\ No newline at end of file diff --git a/archiva-modules/plugins/metadata-store-jcr/pom.xml b/archiva-modules/plugins/metadata-store-jcr/pom.xml index 8fa30860c..8e71c4c89 100644 --- a/archiva-modules/plugins/metadata-store-jcr/pom.xml +++ b/archiva-modules/plugins/metadata-store-jcr/pom.xml @@ -34,6 +34,10 @@ <artifactId>metadata-repository-api</artifactId> </dependency> <dependency> + <groupId>org.apache.archiva</groupId> + <artifactId>metadata-statistics-api</artifactId> + </dependency> + <dependency> <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </dependency> @@ -105,6 +109,10 @@ <directory>src/test/filtered-resources</directory> <filtering>true</filtering> </testResource> + <testResource> + <directory>src/test/resources</directory> + <filtering>false</filtering> + </testResource> </testResources> <plugins> <plugin> diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java index 364930998..2b76db4af 100644 --- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java +++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java @@ -32,9 +32,12 @@ import org.apache.archiva.metadata.model.ProjectMetadata; import org.apache.archiva.metadata.model.ProjectVersionMetadata; import org.apache.archiva.metadata.model.ProjectVersionReference; import org.apache.archiva.metadata.model.Scm; +import org.apache.archiva.metadata.model.maven2.MavenArtifactFacet; import org.apache.archiva.metadata.repository.MetadataRepository; import org.apache.archiva.metadata.repository.MetadataRepositoryException; import org.apache.archiva.metadata.repository.MetadataResolutionException; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsProvider; import org.apache.commons.lang.StringUtils; import org.apache.jackrabbit.commons.JcrUtils; import org.slf4j.Logger; @@ -57,6 +60,7 @@ import javax.jcr.Workspace; import javax.jcr.nodetype.NodeTypeManager; import javax.jcr.nodetype.NodeTypeTemplate; import javax.jcr.query.Query; +import javax.jcr.query.QueryManager; import javax.jcr.query.QueryResult; import javax.jcr.query.Row; import javax.jcr.query.RowIterator; @@ -80,7 +84,7 @@ import java.util.Set; * TODO revise reference storage */ public class JcrMetadataRepository - implements MetadataRepository + implements MetadataRepository,RepositoryStatisticsProvider { private static final String JCR_LAST_MODIFIED = "jcr:lastModified"; @@ -113,7 +117,7 @@ public class JcrMetadataRepository } - static void initialize( Session session ) + public static void initialize( Session session ) throws RepositoryException { @@ -1694,4 +1698,81 @@ public class JcrMetadataRepository } return this.jcrSession; } + + @Override + public void populateStatistics( MetadataRepository repository, String repositoryId, + RepositoryStatistics repositoryStatistics ) + throws MetadataRepositoryException + { + if (!(repository instanceof JcrMetadataRepository)) { + throw new MetadataRepositoryException( "The statistics population is only possible for JcrMetdataRepository implementations" ); + } + Session session = (Session) repository.obtainAccess( Session.class ); + // TODO: these may be best as running totals, maintained by observations on the properties in JCR + + try + { + QueryManager queryManager = session.getWorkspace().getQueryManager(); + + // TODO: JCR-SQL2 query will not complete on a large repo in Jackrabbit 2.2.0 - see JCR-2835 + // Using the JCR-SQL2 variants gives + // "org.apache.lucene.search.BooleanQuery$TooManyClauses: maxClauseCount is set to 1024" +// String whereClause = "WHERE ISDESCENDANTNODE([/repositories/" + repositoryId + "/content])"; +// Query query = queryManager.createQuery( "SELECT size FROM [archiva:artifact] " + whereClause, +// Query.JCR_SQL2 ); + String whereClause = "WHERE jcr:path LIKE '/repositories/" + repositoryId + "/content/%'"; + Query query = queryManager.createQuery( "SELECT size FROM archiva:artifact " + whereClause, Query.SQL ); + + QueryResult queryResult = query.execute(); + + Map<String, Integer> totalByType = new HashMap<>(); + long totalSize = 0, totalArtifacts = 0; + for ( Row row : JcrUtils.getRows( queryResult ) ) + { + Node n = row.getNode(); + totalSize += row.getValue( "size" ).getLong(); + + String type; + if ( n.hasNode( MavenArtifactFacet.FACET_ID ) ) + { + Node facetNode = n.getNode( MavenArtifactFacet.FACET_ID ); + type = facetNode.getProperty( "type" ).getString(); + } + else + { + type = "Other"; + } + Integer prev = totalByType.get( type ); + totalByType.put( type, prev != null ? prev + 1 : 1 ); + + totalArtifacts++; + } + + repositoryStatistics.setTotalArtifactCount( totalArtifacts ); + repositoryStatistics.setTotalArtifactFileSize( totalSize ); + for ( Map.Entry<String, Integer> entry : totalByType.entrySet() ) + { + System.out.println("Setting count for type: "+entry.getKey()+" = "+entry.getValue()); + repositoryStatistics.setTotalCountForType( entry.getKey(), entry.getValue() ); + } + + // The query ordering is a trick to ensure that the size is correct, otherwise due to lazy init it will be -1 +// query = queryManager.createQuery( "SELECT * FROM [archiva:project] " + whereClause, Query.JCR_SQL2 ); + query = queryManager.createQuery( "SELECT * FROM archiva:project " + whereClause + " ORDER BY jcr:score", + Query.SQL ); + repositoryStatistics.setTotalProjectCount( query.execute().getRows().getSize() ); + +// query = queryManager.createQuery( +// "SELECT * FROM [archiva:namespace] " + whereClause + " AND namespace IS NOT NULL", Query.JCR_SQL2 ); + query = queryManager.createQuery( + "SELECT * FROM archiva:namespace " + whereClause + " AND namespace IS NOT NULL ORDER BY jcr:score", + Query.SQL ); + repositoryStatistics.setTotalGroupCount( query.execute().getRows().getSize() ); + } + catch ( RepositoryException e ) + { + throw new MetadataRepositoryException( e.getMessage(), e ); + } + } + } diff --git a/archiva-modules/plugins/repository-statistics/src/test/java/org/apache/archiva/metadata/repository/stats/JcrRepositoryStatisticsGatheringTest.java b/archiva-modules/plugins/metadata-store-jcr/src/test/java/org/apache/archiva/metadata/repository/stats/JcrRepositoryStatisticsGatheringTest.java index 1a46b8727..63bfffb6f 100644 --- a/archiva-modules/plugins/repository-statistics/src/test/java/org/apache/archiva/metadata/repository/stats/JcrRepositoryStatisticsGatheringTest.java +++ b/archiva-modules/plugins/metadata-store-jcr/src/test/java/org/apache/archiva/metadata/repository/stats/JcrRepositoryStatisticsGatheringTest.java @@ -20,16 +20,32 @@ package org.apache.archiva.metadata.repository.stats; */ import junit.framework.TestCase; -import org.apache.archiva.metadata.repository.MetadataRepository; +import org.apache.archiva.metadata.model.MetadataFacetFactory; +import org.apache.archiva.metadata.repository.AbstractMetadataRepositoryTest; import org.apache.archiva.metadata.repository.RepositorySessionFactory; +import org.apache.archiva.metadata.repository.jcr.JcrMetadataRepository; +import org.apache.archiva.metadata.repository.stats.model.DefaultRepositoryStatistics; +import org.apache.archiva.test.utils.ArchivaBlockJUnit4ClassRunner; +import org.apache.archiva.test.utils.ArchivaSpringJUnit4ClassRunner; import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.commons.JcrUtils; import org.apache.jackrabbit.core.TransientRepository; +import org.apache.jackrabbit.core.config.RepositoryConfig; +import org.apache.regexp.RE; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.context.ApplicationContext; +import org.springframework.test.context.ContextConfiguration; import javax.inject.Inject; +import javax.inject.Named; import javax.jcr.ImportUUIDBehavior; import javax.jcr.NamespaceRegistry; import javax.jcr.Node; +import javax.jcr.Repository; import javax.jcr.RepositoryException; import javax.jcr.Session; import javax.jcr.SimpleCredentials; @@ -40,19 +56,15 @@ import java.io.File; import java.io.IOException; import java.util.Calendar; import java.util.Date; +import java.util.Map; import java.util.zip.GZIPInputStream; -import org.apache.archiva.test.utils.ArchivaBlockJUnit4ClassRunner; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.*; +@RunWith( ArchivaSpringJUnit4ClassRunner.class ) +@ContextConfiguration( locations = { "classpath*:/META-INF/spring-context.xml", "classpath*:/spring-context.xml" } ) +public class JcrRepositoryStatisticsGatheringTest extends TestCase -@RunWith( ArchivaBlockJUnit4ClassRunner.class ) -public class JcrRepositoryStatisticsGatheringTest - extends TestCase { private static final int TOTAL_FILE_COUNT = 1000; @@ -60,50 +72,56 @@ public class JcrRepositoryStatisticsGatheringTest private static final String TEST_REPO = "test-repo"; - private RepositoryStatisticsManager repositoryStatisticsManager; - - private MetadataRepository metadataRepository; + JcrMetadataRepository repository; @Inject private RepositorySessionFactory repositorySessionFactory; - private Session session; + @Inject + private ApplicationContext applicationContext; + + @Inject + @Named("repository") + Repository jcrRepository; + + Session session; + + + - @Override @Before public void setUp() throws Exception { - super.setUp(); - File confFile = new File( "src/test/repository.xml" ); - File dir = new File( "target/jcr" ); - FileUtils.deleteDirectory( dir ); + File directory = new File( "target/test-repositories" ); + if ( directory.exists() ) + { + FileUtils.deleteDirectory( directory ); + } - assertTrue( confFile.exists() ); - assertFalse( dir.exists() ); + Map<String, MetadataFacetFactory> factories = AbstractMetadataRepositoryTest.createTestMetadataFacetFactories(); - TransientRepository repository = new TransientRepository( confFile, dir ); - session = repository.login( new SimpleCredentials( "username", "password".toCharArray() ) ); + assertNotNull( jcrRepository ); + // TODO: probably don't need to use Spring for this + JcrMetadataRepository jcrMetadataRepository = new JcrMetadataRepository( factories, jcrRepository ); - // TODO: perhaps have an archiva-jcr-utils module shared by these plugins that does this and can contain - // structure information - Workspace workspace = session.getWorkspace(); - NamespaceRegistry registry = workspace.getNamespaceRegistry(); - registry.registerNamespace( "archiva", "http://archiva.apache.org/jcr/" ); + try + { + session = jcrMetadataRepository.getJcrSession(); - NodeTypeManager nodeTypeManager = workspace.getNodeTypeManager(); - registerMixinNodeType( nodeTypeManager, "archiva:namespace" ); - registerMixinNodeType( nodeTypeManager, "archiva:project" ); - registerMixinNodeType( nodeTypeManager, "archiva:projectVersion" ); - registerMixinNodeType( nodeTypeManager, "archiva:artifact" ); - registerMixinNodeType( nodeTypeManager, "archiva:facet" ); + // set up namespaces, etc. + JcrMetadataRepository.initialize( session ); - metadataRepository = mock( MetadataRepository.class ); - when( metadataRepository.canObtainAccess( Session.class ) ).thenReturn( true ); - when( metadataRepository.obtainAccess( Session.class ) ).thenReturn( session ); + // removing content is faster than deleting and re-copying the files from target/jcr + session.getRootNode().getNode( "repositories" ).remove(); + } + catch ( RepositoryException e ) + { + // ignore + } - repositoryStatisticsManager = new DefaultRepositoryStatisticsManager(); + this.repository = jcrMetadataRepository; } private static void registerMixinNodeType( NodeTypeManager nodeTypeManager, String type ) @@ -115,17 +133,15 @@ public class JcrRepositoryStatisticsGatheringTest nodeTypeManager.registerNodeType( nodeType, false ); } - @Override @After public void tearDown() throws Exception { - if ( session != null ) + if (repository!=null) { - session.logout(); + repository.close( ); } - super.tearDown(); } @Test @@ -140,10 +156,15 @@ public class JcrRepositoryStatisticsGatheringTest loadContentIntoRepo( TEST_REPO ); loadContentIntoRepo( "another-repo" ); - repositoryStatisticsManager.addStatisticsAfterScan( metadataRepository, TEST_REPO, startTime, endTime, - TOTAL_FILE_COUNT, NEW_FILE_COUNT ); + DefaultRepositoryStatistics testedStatistics = new DefaultRepositoryStatistics(); + testedStatistics.setNewFileCount( NEW_FILE_COUNT ); + testedStatistics.setTotalFileCount( TOTAL_FILE_COUNT ); + testedStatistics.setScanStartTime( startTime ); + testedStatistics.setScanEndTime( endTime ); - RepositoryStatistics expectedStatistics = new RepositoryStatistics(); + repository.populateStatistics( repository, TEST_REPO, testedStatistics ); + + DefaultRepositoryStatistics expectedStatistics = new DefaultRepositoryStatistics(); expectedStatistics.setNewFileCount( NEW_FILE_COUNT ); expectedStatistics.setTotalFileCount( TOTAL_FILE_COUNT ); expectedStatistics.setScanEndTime( endTime ); @@ -161,7 +182,26 @@ public class JcrRepositoryStatisticsGatheringTest expectedStatistics.setTotalCountForType( "pom", 144 ); expectedStatistics.setRepositoryId( TEST_REPO ); - verify( metadataRepository ).addMetadataFacet( TEST_REPO, expectedStatistics ); + System.out.println(testedStatistics.getTotalCountForType()); + + assertEquals( NEW_FILE_COUNT, testedStatistics.getNewFileCount()); + assertEquals( TOTAL_FILE_COUNT, testedStatistics.getTotalFileCount() ); + assertEquals( endTime, testedStatistics.getScanEndTime() ); + assertEquals( startTime, testedStatistics.getScanStartTime() ); + assertEquals( 95954585, testedStatistics.getTotalArtifactFileSize() ); + assertEquals( 269, testedStatistics.getTotalArtifactCount() ); + assertEquals( 1, testedStatistics.getTotalGroupCount() ); + assertEquals( 43, testedStatistics.getTotalProjectCount() ); + assertEquals( 1, testedStatistics.getTotalCountForType( "zip" ) ); + assertEquals( 1, testedStatistics.getTotalCountForType( "gz" ) ); + assertEquals( 10, testedStatistics.getTotalCountForType( "java-source" ) ); + assertEquals( 108, testedStatistics.getTotalCountForType( "jar" ) ); + assertEquals( 3, testedStatistics.getTotalCountForType( "xml" ) ); + assertEquals( 2, testedStatistics.getTotalCountForType( "war" ) ); + assertEquals( 144, testedStatistics.getTotalCountForType( "pom" ) ); + assertEquals( 10, testedStatistics.getTotalCountForType( "java-source" ) ); + + } private void loadContentIntoRepo( String repoId ) diff --git a/archiva-modules/plugins/metadata-store-jcr/src/test/resources/artifacts.xml.gz b/archiva-modules/plugins/metadata-store-jcr/src/test/resources/artifacts.xml.gz Binary files differnew file mode 100644 index 000000000..0d8a41e3b --- /dev/null +++ b/archiva-modules/plugins/metadata-store-jcr/src/test/resources/artifacts.xml.gz diff --git a/archiva-modules/plugins/repository-statistics/pom.xml b/archiva-modules/plugins/repository-statistics/pom.xml index 2d20acacb..8265ed1eb 100644 --- a/archiva-modules/plugins/repository-statistics/pom.xml +++ b/archiva-modules/plugins/repository-statistics/pom.xml @@ -34,6 +34,10 @@ </dependency> <dependency> <groupId>org.apache.archiva</groupId> + <artifactId>metadata-statistics-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.archiva</groupId> <artifactId>metadata-model-maven2</artifactId> </dependency> <dependency> @@ -41,14 +45,6 @@ <artifactId>slf4j-api</artifactId> </dependency> <dependency> - <groupId>javax.jcr</groupId> - <artifactId>jcr</artifactId> - </dependency> - <dependency> - <groupId>org.apache.jackrabbit</groupId> - <artifactId>jackrabbit-jcr-commons</artifactId> - </dependency> - <dependency> <groupId>org.apache.archiva</groupId> <artifactId>archiva-test-utils</artifactId> <version>${project.version}</version> @@ -60,17 +56,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.jackrabbit</groupId> - <artifactId>jackrabbit-core</artifactId> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>jcl-over-slf4j</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> diff --git a/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/DefaultRepositoryStatisticsManager.java b/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/DefaultRepositoryStatisticsManager.java index b05bea767..f7c4dcc84 100644 --- a/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/DefaultRepositoryStatisticsManager.java +++ b/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/DefaultRepositoryStatisticsManager.java @@ -19,13 +19,14 @@ package org.apache.archiva.metadata.repository.stats; * under the License. */ -import org.apache.archiva.metadata.model.ArtifactMetadata; -import org.apache.archiva.metadata.model.maven2.MavenArtifactFacet; import org.apache.archiva.metadata.repository.MetadataRepository; import org.apache.archiva.metadata.repository.MetadataRepositoryException; -import org.apache.archiva.metadata.repository.MetadataResolutionException; +import org.apache.archiva.metadata.repository.stats.model.DefaultRepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsManager; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatisticsProvider; +import org.apache.archiva.metadata.repository.stats.model.RepositoryWalkingStatisticsProvider; import org.apache.commons.lang.time.StopWatch; -import org.apache.jackrabbit.commons.JcrUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -33,20 +34,10 @@ import org.springframework.stereotype.Service; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.TimeZone; -import javax.jcr.Node; -import javax.jcr.RepositoryException; -import javax.jcr.Session; -import javax.jcr.query.Query; -import javax.jcr.query.QueryManager; -import javax.jcr.query.QueryResult; -import javax.jcr.query.Row; /** * @@ -59,11 +50,13 @@ public class DefaultRepositoryStatisticsManager private static final TimeZone UTC_TIME_ZONE = TimeZone.getTimeZone( "UTC" ); + private RepositoryWalkingStatisticsProvider walkingProvider = new RepositoryWalkingStatisticsProvider(); + @Override public boolean hasStatistics( MetadataRepository metadataRepository, String repositoryId ) throws MetadataRepositoryException { - return metadataRepository.hasMetadataFacet( repositoryId, RepositoryStatistics.FACET_ID ); + return metadataRepository.hasMetadataFacet( repositoryId, DefaultRepositoryStatistics.FACET_ID ); } @Override @@ -73,7 +66,7 @@ public class DefaultRepositoryStatisticsManager StopWatch stopWatch = new StopWatch(); stopWatch.start(); // TODO: consider a more efficient implementation that directly gets the last one from the content repository - List<String> scans = metadataRepository.getMetadataFacets( repositoryId, RepositoryStatistics.FACET_ID ); + List<String> scans = metadataRepository.getMetadataFacets( repositoryId, DefaultRepositoryStatistics.FACET_ID ); if ( scans == null ) { return null; @@ -95,50 +88,12 @@ public class DefaultRepositoryStatisticsManager } } - private void walkRepository( MetadataRepository metadataRepository, RepositoryStatistics stats, String repositoryId, - String ns ) - throws MetadataResolutionException - { - for ( String namespace : metadataRepository.getNamespaces( repositoryId, ns ) ) - { - walkRepository( metadataRepository, stats, repositoryId, ns + "." + namespace ); - } - - Collection<String> projects = metadataRepository.getProjects( repositoryId, ns ); - if ( !projects.isEmpty() ) - { - stats.setTotalGroupCount( stats.getTotalGroupCount() + 1 ); - stats.setTotalProjectCount( stats.getTotalProjectCount() + projects.size() ); - - for ( String project : projects ) - { - for ( String version : metadataRepository.getProjectVersions( repositoryId, ns, project ) ) - { - for ( ArtifactMetadata artifact : metadataRepository.getArtifacts( repositoryId, ns, project, - version ) ) - { - stats.setTotalArtifactCount( stats.getTotalArtifactCount() + 1 ); - stats.setTotalArtifactFileSize( stats.getTotalArtifactFileSize() + artifact.getSize() ); - - MavenArtifactFacet facet = - (MavenArtifactFacet) artifact.getFacet( MavenArtifactFacet.FACET_ID ); - if ( facet != null ) - { - String type = facet.getType(); - stats.setTotalCountForType( type, stats.getTotalCountForType( type ) + 1 ); - } - } - } - } - } - } - @Override public void addStatisticsAfterScan( MetadataRepository metadataRepository, String repositoryId, Date startTime, Date endTime, long totalFiles, long newFiles ) throws MetadataRepositoryException { - RepositoryStatistics repositoryStatistics = new RepositoryStatistics(); + DefaultRepositoryStatistics repositoryStatistics = new DefaultRepositoryStatistics(); repositoryStatistics.setRepositoryId( repositoryId ); repositoryStatistics.setScanStartTime( startTime ); repositoryStatistics.setScanEndTime( endTime ); @@ -154,23 +109,14 @@ public class DefaultRepositoryStatisticsManager long startGather = System.currentTimeMillis(); - // FIXME what about other implementations ? - - if ( metadataRepository.canObtainAccess( Session.class ) ) + if ( metadataRepository instanceof RepositoryStatisticsProvider) { - // TODO: this is currently very raw and susceptible to changes in content structure. Should we instead - // depend directly on the plugin and interrogate the JCR repository's knowledge of the structure? - populateStatisticsFromJcr( (Session) metadataRepository.obtainAccess( Session.class ), repositoryId, - repositoryStatistics ); + ((RepositoryStatisticsProvider)metadataRepository).populateStatistics( metadataRepository, + repositoryId, repositoryStatistics); } else { - // TODO: - // if the file repository is used more permanently, we may seek a more efficient mechanism - e.g. we could - // build an index, or store the aggregate information and update it on the fly. We can perhaps even walk - // but retrieve less information to speed it up. In the mean time, we walk the repository using the - // standard APIs - populateStatisticsFromRepositoryWalk( metadataRepository, repositoryId, repositoryStatistics ); + walkingProvider.populateStatistics( metadataRepository, repositoryId, repositoryStatistics ); } log.info( "Gathering statistics executed in {} ms", ( System.currentTimeMillis() - startGather ) ); @@ -178,98 +124,11 @@ public class DefaultRepositoryStatisticsManager metadataRepository.addMetadataFacet( repositoryId, repositoryStatistics ); } - private void populateStatisticsFromJcr( Session session, String repositoryId, - RepositoryStatistics repositoryStatistics ) - throws MetadataRepositoryException - { - // TODO: these may be best as running totals, maintained by observations on the properties in JCR - - try - { - QueryManager queryManager = session.getWorkspace().getQueryManager(); - - // TODO: JCR-SQL2 query will not complete on a large repo in Jackrabbit 2.2.0 - see JCR-2835 - // Using the JCR-SQL2 variants gives - // "org.apache.lucene.search.BooleanQuery$TooManyClauses: maxClauseCount is set to 1024" -// String whereClause = "WHERE ISDESCENDANTNODE([/repositories/" + repositoryId + "/content])"; -// Query query = queryManager.createQuery( "SELECT size FROM [archiva:artifact] " + whereClause, -// Query.JCR_SQL2 ); - String whereClause = "WHERE jcr:path LIKE '/repositories/" + repositoryId + "/content/%'"; - Query query = queryManager.createQuery( "SELECT size FROM archiva:artifact " + whereClause, Query.SQL ); - - QueryResult queryResult = query.execute(); - - Map<String, Integer> totalByType = new HashMap<>(); - long totalSize = 0, totalArtifacts = 0; - for ( Row row : JcrUtils.getRows( queryResult ) ) - { - Node n = row.getNode(); - totalSize += row.getValue( "size" ).getLong(); - - String type; - if ( n.hasNode( MavenArtifactFacet.FACET_ID ) ) - { - Node facetNode = n.getNode( MavenArtifactFacet.FACET_ID ); - type = facetNode.getProperty( "type" ).getString(); - } - else - { - type = "Other"; - } - Integer prev = totalByType.get( type ); - totalByType.put( type, prev != null ? prev + 1 : 1 ); - - totalArtifacts++; - } - - repositoryStatistics.setTotalArtifactCount( totalArtifacts ); - repositoryStatistics.setTotalArtifactFileSize( totalSize ); - for ( Map.Entry<String, Integer> entry : totalByType.entrySet() ) - { - repositoryStatistics.setTotalCountForType( entry.getKey(), entry.getValue() ); - } - - // The query ordering is a trick to ensure that the size is correct, otherwise due to lazy init it will be -1 -// query = queryManager.createQuery( "SELECT * FROM [archiva:project] " + whereClause, Query.JCR_SQL2 ); - query = queryManager.createQuery( "SELECT * FROM archiva:project " + whereClause + " ORDER BY jcr:score", - Query.SQL ); - repositoryStatistics.setTotalProjectCount( query.execute().getRows().getSize() ); - -// query = queryManager.createQuery( -// "SELECT * FROM [archiva:namespace] " + whereClause + " AND namespace IS NOT NULL", Query.JCR_SQL2 ); - query = queryManager.createQuery( - "SELECT * FROM archiva:namespace " + whereClause + " AND namespace IS NOT NULL ORDER BY jcr:score", - Query.SQL ); - repositoryStatistics.setTotalGroupCount( query.execute().getRows().getSize() ); - } - catch ( RepositoryException e ) - { - throw new MetadataRepositoryException( e.getMessage(), e ); - } - } - - private void populateStatisticsFromRepositoryWalk( MetadataRepository metadataRepository, String repositoryId, - RepositoryStatistics repositoryStatistics ) - throws MetadataRepositoryException - { - try - { - for ( String ns : metadataRepository.getRootNamespaces( repositoryId ) ) - { - walkRepository( metadataRepository, repositoryStatistics, repositoryId, ns ); - } - } - catch ( MetadataResolutionException e ) - { - throw new MetadataRepositoryException( e.getMessage(), e ); - } - } - @Override public void deleteStatistics( MetadataRepository metadataRepository, String repositoryId ) throws MetadataRepositoryException { - metadataRepository.removeMetadataFacets( repositoryId, RepositoryStatistics.FACET_ID ); + metadataRepository.removeMetadataFacets( repositoryId, DefaultRepositoryStatistics.FACET_ID ); } @Override @@ -278,7 +137,7 @@ public class DefaultRepositoryStatisticsManager throws MetadataRepositoryException { List<RepositoryStatistics> results = new ArrayList<>(); - List<String> list = metadataRepository.getMetadataFacets( repositoryId, RepositoryStatistics.FACET_ID ); + List<String> list = metadataRepository.getMetadataFacets( repositoryId, DefaultRepositoryStatistics.FACET_ID ); Collections.sort( list, Collections.reverseOrder() ); for ( String name : list ) { @@ -290,7 +149,7 @@ public class DefaultRepositoryStatisticsManager { RepositoryStatistics stats = (RepositoryStatistics) metadataRepository.getMetadataFacet( repositoryId, - RepositoryStatistics.FACET_ID, + DefaultRepositoryStatistics.FACET_ID, name ); results.add( stats ); } @@ -306,7 +165,7 @@ public class DefaultRepositoryStatisticsManager private static SimpleDateFormat createNameFormat() { - SimpleDateFormat fmt = new SimpleDateFormat( RepositoryStatistics.SCAN_TIMESTAMP_FORMAT ); + SimpleDateFormat fmt = new SimpleDateFormat( DefaultRepositoryStatistics.SCAN_TIMESTAMP_FORMAT ); fmt.setTimeZone( UTC_TIME_ZONE ); return fmt; } diff --git a/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsFactory.java b/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsFactory.java index 198464688..ceb911803 100644 --- a/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsFactory.java +++ b/archiva-modules/plugins/repository-statistics/src/main/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsFactory.java @@ -21,6 +21,7 @@ package org.apache.archiva.metadata.repository.stats; import org.apache.archiva.metadata.model.MetadataFacet; import org.apache.archiva.metadata.model.MetadataFacetFactory; +import org.apache.archiva.metadata.repository.stats.model.DefaultRepositoryStatistics; import org.springframework.stereotype.Service; /** @@ -33,12 +34,12 @@ public class RepositoryStatisticsFactory @Override public MetadataFacet createMetadataFacet() { - return new RepositoryStatistics(); + return new DefaultRepositoryStatistics(); } @Override public MetadataFacet createMetadataFacet( String repositoryId, String name ) { - return new RepositoryStatistics(); + return new DefaultRepositoryStatistics(); } }
\ No newline at end of file diff --git a/archiva-modules/plugins/repository-statistics/src/test/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsManagerTest.java b/archiva-modules/plugins/repository-statistics/src/test/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsManagerTest.java index 2ec08e6b5..b0a1742e9 100644 --- a/archiva-modules/plugins/repository-statistics/src/test/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsManagerTest.java +++ b/archiva-modules/plugins/repository-statistics/src/test/java/org/apache/archiva/metadata/repository/stats/RepositoryStatisticsManagerTest.java @@ -23,16 +23,14 @@ import junit.framework.TestCase; import org.apache.archiva.metadata.model.ArtifactMetadata; import org.apache.archiva.metadata.model.maven2.MavenArtifactFacet; import org.apache.archiva.metadata.repository.MetadataRepository; +import org.apache.archiva.metadata.repository.stats.model.DefaultRepositoryStatistics; +import org.apache.archiva.metadata.repository.stats.model.RepositoryStatistics; import org.apache.archiva.test.utils.ArchivaBlockJUnit4ClassRunner; import org.easymock.IMocksControl; - -import static org.easymock.EasyMock.*; - import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import javax.jcr.Session; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -43,6 +41,9 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; +import static org.easymock.EasyMock.createControl; +import static org.easymock.EasyMock.expect; + @RunWith( ArchivaBlockJUnit4ClassRunner.class ) public class RepositoryStatisticsManagerTest extends TestCase @@ -65,7 +66,7 @@ public class RepositoryStatisticsManagerTest private static SimpleDateFormat createTimestampFormat() { - SimpleDateFormat fmt = new SimpleDateFormat( RepositoryStatistics.SCAN_TIMESTAMP_FORMAT ); + SimpleDateFormat fmt = new SimpleDateFormat( DefaultRepositoryStatistics.SCAN_TIMESTAMP_FORMAT ); fmt.setTimeZone( TimeZone.getTimeZone( "UTC" ) ); return fmt; } @@ -90,9 +91,10 @@ public class RepositoryStatisticsManagerTest Date startTime = TIMESTAMP_FORMAT.parse( SECOND_TEST_SCAN ); Date endTime = new Date( startTime.getTime() + 60000 ); - RepositoryStatistics stats = new RepositoryStatistics(); - stats.setScanStartTime( startTime ); - stats.setScanEndTime( endTime ); + DefaultRepositoryStatistics defStats = new DefaultRepositoryStatistics(); + defStats.setScanStartTime( startTime ); + defStats.setScanEndTime( endTime ); + RepositoryStatistics stats = defStats; stats.setTotalArtifactFileSize( 1314527915L ); stats.setNewFileCount( 123 ); stats.setTotalArtifactCount( 10386 ); @@ -101,10 +103,10 @@ public class RepositoryStatisticsManagerTest stats.setTotalFileCount( 56229 ); - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( Arrays.asList( FIRST_TEST_SCAN, SECOND_TEST_SCAN ) ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, SECOND_TEST_SCAN ) ).andReturn( stats ); metadataRepositoryControl.replay(); @@ -129,7 +131,7 @@ public class RepositoryStatisticsManagerTest throws Exception { - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( Collections.<String>emptyList() ); metadataRepositoryControl.replay(); @@ -152,14 +154,12 @@ public class RepositoryStatisticsManagerTest metadataRepository.addMetadataFacet( TEST_REPO_ID, stats ); - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( Arrays.asList( stats.getName() ) ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, stats.getName() ) ).andReturn( stats ); - expect( metadataRepository.canObtainAccess( Session.class ) ).andReturn( false ); - metadataRepositoryControl.replay(); repositoryStatisticsManager.addStatisticsAfterScan( metadataRepository, TEST_REPO_ID, startTime, current, 56345, @@ -188,25 +188,24 @@ public class RepositoryStatisticsManagerTest Date current = new Date(); Date startTime1 = new Date( current.getTime() - 12345 ); - RepositoryStatistics stats1 = createTestStats( startTime1, new Date( current.getTime() - 6000 ) ); + DefaultRepositoryStatistics stats1 = createTestStats( startTime1, new Date( current.getTime() - 6000 ) ); metadataRepository.addMetadataFacet( TEST_REPO_ID, stats1 ); Date startTime2 = new Date( current.getTime() - 3000 ); - RepositoryStatistics stats2 = createTestStats( startTime2, current ); + DefaultRepositoryStatistics stats2 = createTestStats( startTime2, current ); metadataRepository.addMetadataFacet( TEST_REPO_ID, stats2 ); - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( Arrays.asList( stats1.getName(), stats2.getName() ) ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, stats2.getName() ) ).andReturn( stats2 ); - metadataRepository.removeMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ); + metadataRepository.removeMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ); - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( Collections.<String>emptyList() ); - expect( metadataRepository.canObtainAccess( Session.class ) ).andReturn( false ).times( 2 ); metadataRepositoryControl.replay(); repositoryStatisticsManager.addStatisticsAfterScan( metadataRepository, TEST_REPO_ID, startTime1, @@ -228,9 +227,9 @@ public class RepositoryStatisticsManagerTest throws Exception { - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( Collections.<String>emptyList() ).times( 2 ); - metadataRepository.removeMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ); + metadataRepository.removeMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ); metadataRepositoryControl.replay(); @@ -257,15 +256,14 @@ public class RepositoryStatisticsManagerTest ArrayList<String> keys = new ArrayList<>( statsCreated.keySet() ); - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( keys ); + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( keys ); // only match the middle one String key = keys.get( 1 ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, key ) ).andReturn( + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, key ) ).andReturn( statsCreated.get( key ) ); - expect( metadataRepository.canObtainAccess( Session.class ) ).andReturn( false ).times( 3 ); metadataRepositoryControl.replay(); @@ -301,19 +299,18 @@ public class RepositoryStatisticsManagerTest List<String> keys = new ArrayList<>( statsCreated.keySet() ); - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( keys ); + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( keys ); String key = keys.get( 1 ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, key ) ).andReturn( + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, key ) ).andReturn( statsCreated.get( key ) ); key = keys.get( 2 ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, key ) ).andReturn( + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, key ) ).andReturn( statsCreated.get( key ) ); - expect( metadataRepository.canObtainAccess( Session.class ) ).andReturn( false ).times( 3 ); metadataRepositoryControl.replay(); @@ -349,19 +346,17 @@ public class RepositoryStatisticsManagerTest List<String> keys = new ArrayList<>( statsCreated.keySet() ); - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( keys ); + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( keys ); String key = keys.get( 0 ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, key ) ).andReturn( + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, key ) ).andReturn( statsCreated.get( key ) ); key = keys.get( 1 ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, key ) ).andReturn( + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, key ) ).andReturn( statsCreated.get( key ) ); - expect( metadataRepository.canObtainAccess( Session.class ) ).andReturn( false ).times( 3 ); - metadataRepositoryControl.replay(); for ( RepositoryStatistics stats : statsCreated.values() ) @@ -397,23 +392,21 @@ public class RepositoryStatisticsManagerTest ArrayList<String> keys = new ArrayList<>( statsCreated.keySet() ); - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( keys ); + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( keys ); String key = keys.get( 0 ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, key ) ).andReturn( + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, key ) ).andReturn( statsCreated.get( key ) ); key = keys.get( 1 ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, key ) ).andReturn( + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, key ) ).andReturn( statsCreated.get( key ) ); key = keys.get( 2 ); - expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, RepositoryStatistics.FACET_ID, key ) ).andReturn( + expect( metadataRepository.getMetadataFacet( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID, key ) ).andReturn( statsCreated.get( key ) ); - expect( metadataRepository.canObtainAccess( Session.class ) ).andReturn( false ).times( 3 ); - metadataRepositoryControl.replay(); for ( RepositoryStatistics stats : statsCreated.values() ) @@ -449,9 +442,7 @@ public class RepositoryStatisticsManagerTest ArrayList<String> keys = new ArrayList<>( statsCreated.keySet() ); - expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, RepositoryStatistics.FACET_ID ) ).andReturn( keys ); - - expect( metadataRepository.canObtainAccess( Session.class ) ).andReturn( false ).times( 3 ); + expect( metadataRepository.getMetadataFacets( TEST_REPO_ID, DefaultRepositoryStatistics.FACET_ID ) ).andReturn( keys ); metadataRepositoryControl.replay(); @@ -475,7 +466,7 @@ public class RepositoryStatisticsManagerTest private void addStats( Date startTime, Date endTime ) throws Exception { - RepositoryStatistics stats = createTestStats( startTime, endTime ); + DefaultRepositoryStatistics stats = createTestStats( startTime, endTime ); metadataRepository.addMetadataFacet( TEST_REPO_ID, stats ); statsCreated.put( stats.getName(), stats ); } @@ -498,9 +489,9 @@ public class RepositoryStatisticsManagerTest return metadata; } - private RepositoryStatistics createTestStats( Date startTime, Date endTime ) + private DefaultRepositoryStatistics createTestStats( Date startTime, Date endTime ) { - RepositoryStatistics stats = new RepositoryStatistics(); + DefaultRepositoryStatistics stats = new DefaultRepositoryStatistics(); stats.setRepositoryId( TEST_REPO_ID ); stats.setScanStartTime( startTime ); stats.setScanEndTime( endTime ); @@ -195,6 +195,11 @@ </dependency> <dependency> <groupId>org.apache.archiva</groupId> + <artifactId>metadata-statistics-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.archiva</groupId> <artifactId>metadata-repository-api</artifactId> <version>${project.version}</version> <classifier>tests</classifier> |