From 15549c85b0a02b4cf1aa9d51a07e72a500c6a3d4 Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Mon, 12 Mar 2012 22:25:13 +0000 Subject: [PATCH] [MRM-1580] system status page current scanner stats. git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1299905 13f79535-47bb-0310-9956-ffa450edef68 --- .../scanner/RepositoryScannerInstance.java | 29 +++-- .../api/model/ConsumerScanningStatistics.java | 71 +++++++++++ .../model/RepositoryScannerStatistics.java | 119 ++++++++++++++++++ .../api/services/SystemStatusService.java | 8 ++ .../services/DefaultSystemStatusService.java | 59 ++++++++- .../main/webapp/js/archiva/general-admin.js | 40 ++++++ .../js/archiva/templates/general-admin.html | 29 ++++- 7 files changed, 336 insertions(+), 19 deletions(-) create mode 100644 archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/ConsumerScanningStatistics.java create mode 100644 archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/RepositoryScannerStatistics.java diff --git a/archiva-modules/archiva-base/archiva-repository-scanner/src/main/java/org/apache/archiva/repository/scanner/RepositoryScannerInstance.java b/archiva-modules/archiva-base/archiva-repository-scanner/src/main/java/org/apache/archiva/repository/scanner/RepositoryScannerInstance.java index 059d6b528..80a6fb730 100644 --- a/archiva-modules/archiva-base/archiva-repository-scanner/src/main/java/org/apache/archiva/repository/scanner/RepositoryScannerInstance.java +++ b/archiva-modules/archiva-base/archiva-repository-scanner/src/main/java/org/apache/archiva/repository/scanner/RepositoryScannerInstance.java @@ -20,6 +20,10 @@ package org.apache.archiva.repository.scanner; */ import org.apache.archiva.admin.model.beans.ManagedRepository; +import org.apache.archiva.common.utils.BaseFile; +import org.apache.archiva.consumers.InvalidRepositoryContentConsumer; +import org.apache.archiva.consumers.KnownRepositoryContentConsumer; +import org.apache.archiva.consumers.functors.ConsumerWantsFilePredicate; import org.apache.archiva.repository.scanner.functors.ConsumerProcessFileClosure; import org.apache.archiva.repository.scanner.functors.TriggerBeginScanClosure; import org.apache.archiva.repository.scanner.functors.TriggerScanCompletedClosure; @@ -27,10 +31,6 @@ import org.apache.commons.collections.Closure; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.functors.IfClosure; import org.apache.commons.lang.SystemUtils; -import org.apache.archiva.common.utils.BaseFile; -import org.apache.archiva.consumers.InvalidRepositoryContentConsumer; -import org.apache.archiva.consumers.KnownRepositoryContentConsumer; -import org.apache.archiva.consumers.functors.ConsumerWantsFilePredicate; import org.codehaus.plexus.util.DirectoryWalkListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,8 +40,9 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; + /** - * RepositoryScannerInstance + * RepositoryScannerInstance * * @version $Id$ */ @@ -49,7 +50,7 @@ public class RepositoryScannerInstance implements DirectoryWalkListener { private Logger log = LoggerFactory.getLogger( RepositoryScannerInstance.class ); - + /** * Consumers that process known content. */ @@ -82,8 +83,8 @@ public class RepositoryScannerInstance this.knownConsumers = knownConsumerList; this.invalidConsumers = invalidConsumerList; - consumerTimings = new HashMap(); - consumerCounts = new HashMap(); + consumerTimings = new HashMap(); + consumerCounts = new HashMap(); this.consumerProcessFile = new ConsumerProcessFileClosure(); consumerProcessFile.setExecuteOnEntireRepo( true ); @@ -95,7 +96,8 @@ public class RepositoryScannerInstance stats = new RepositoryScanStatistics(); stats.setRepositoryId( repository.getId() ); - Closure triggerBeginScan = new TriggerBeginScanClosure( repository, new Date( System.currentTimeMillis() ), true ); + Closure triggerBeginScan = + new TriggerBeginScanClosure( repository, new Date( System.currentTimeMillis() ), true ); CollectionUtils.forAllDo( knownConsumerList, triggerBeginScan ); CollectionUtils.forAllDo( invalidConsumerList, triggerBeginScan ); @@ -108,7 +110,8 @@ public class RepositoryScannerInstance public RepositoryScannerInstance( ManagedRepository repository, List knownContentConsumers, - List invalidContentConsumers, long changesSince ) + List invalidContentConsumers, + long changesSince ) { this( repository, knownContentConsumers, invalidContentConsumers ); @@ -134,7 +137,7 @@ public class RepositoryScannerInstance public void directoryWalkStarting( File basedir ) { - log.info( "Walk Started: [" + this.repository.getId() + "] " + this.repository.getLocation() ); + log.info( "Walk Started: [{}] {}", this.repository.getId(), this.repository.getLocation() ); stats.triggerStart(); } @@ -175,7 +178,7 @@ public class RepositoryScannerInstance stats.setConsumerTimings( consumerTimings ); stats.setConsumerCounts( consumerCounts ); - log.info( "Walk Finished: [" + this.repository.getId() + "] " + this.repository.getLocation() ); + log.info( "Walk Finished: [{}] {}", this.repository.getId(), this.repository.getLocation() ); stats.triggerFinished(); } @@ -186,7 +189,7 @@ public class RepositoryScannerInstance { log.debug( "Repository Scanner: {}", message ); } - + public ManagedRepository getRepository() { return repository; diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/ConsumerScanningStatistics.java b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/ConsumerScanningStatistics.java new file mode 100644 index 000000000..1df24589b --- /dev/null +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/ConsumerScanningStatistics.java @@ -0,0 +1,71 @@ +package org.apache.archiva.rest.api.model; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; + +/** + * @author Olivier Lamy + */ +@XmlRootElement( name = "consumerScanningStatistics" ) +public class ConsumerScanningStatistics + implements Serializable +{ + private String consumerKey; + + private long count; + + private long time; + + public ConsumerScanningStatistics() + { + // no op + } + + public String getConsumerKey() + { + return consumerKey; + } + + public void setConsumerKey( String consumerKey ) + { + this.consumerKey = consumerKey; + } + + public long getCount() + { + return count; + } + + public void setCount( long count ) + { + this.count = count; + } + + public long getTime() + { + return time; + } + + public void setTime( long time ) + { + this.time = time; + } +} diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/RepositoryScannerStatistics.java b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/RepositoryScannerStatistics.java new file mode 100644 index 000000000..d7e149e6f --- /dev/null +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/RepositoryScannerStatistics.java @@ -0,0 +1,119 @@ +package org.apache.archiva.rest.api.model; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.admin.model.beans.ManagedRepository; + +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.List; + +/** + * @author Olivier Lamy + */ +@XmlRootElement( name = "repositoryScannerStatistics" ) +public class RepositoryScannerStatistics + implements Serializable +{ + private ManagedRepository managedRepository; + + //private RepositoryScanStatistics repositoryScanStatistics; + + private List consumerScanningStatistics; + + private long totalFileCount = 0; + + private long newFileCount = 0; + + public RepositoryScannerStatistics() + { + // no op + } + + public ManagedRepository getManagedRepository() + { + return managedRepository; + } + + public void setManagedRepository( ManagedRepository managedRepository ) + { + this.managedRepository = managedRepository; + } + + /* + public RepositoryScanStatistics getRepositoryScanStatistics() + { + return repositoryScanStatistics; + } + + public void setRepositoryScanStatistics( RepositoryScanStatistics repositoryScanStatistics ) + { + this.repositoryScanStatistics = repositoryScanStatistics; + }*/ + + /*public Map getConsumerCounts() + { + return consumerCounts; + } + + public void setConsumerCounts( Map consumerCounts ) + { + this.consumerCounts = consumerCounts; + } + + public Map getConsumerTimings() + { + return consumerTimings; + } + + public void setConsumerTimings( Map consumerTimings ) + { + this.consumerTimings = consumerTimings; + } */ + + public List getConsumerScanningStatistics() + { + return consumerScanningStatistics; + } + + public void setConsumerScanningStatistics( List consumerScanningStatistics ) + { + this.consumerScanningStatistics = consumerScanningStatistics; + } + + public long getTotalFileCount() + { + return totalFileCount; + } + + public void setTotalFileCount( long totalFileCount ) + { + this.totalFileCount = totalFileCount; + } + + public long getNewFileCount() + { + return newFileCount; + } + + public void setNewFileCount( long newFileCount ) + { + this.newFileCount = newFileCount; + } +} diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/services/SystemStatusService.java b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/services/SystemStatusService.java index 8e0c809b3..8ec56b2cc 100644 --- a/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/services/SystemStatusService.java +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/services/SystemStatusService.java @@ -20,6 +20,7 @@ package org.apache.archiva.rest.api.services; import org.apache.archiva.rest.api.model.CacheEntry; import org.apache.archiva.rest.api.model.QueueEntry; +import org.apache.archiva.rest.api.model.RepositoryScannerStatistics; import org.apache.archiva.security.common.ArchivaRoleConstants; import org.codehaus.plexus.redback.authorization.RedbackAuthorization; @@ -80,4 +81,11 @@ public interface SystemStatusService throws ArchivaRestServiceException; + @Path( "repositoryScannerStatistics" ) + @GET + @Produces( { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML } ) + @RedbackAuthorization( permissions = ArchivaRoleConstants.OPERATION_MANAGE_CONFIGURATION ) + List getRepositoryScannerStatistics() + throws ArchivaRestServiceException; + } diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultSystemStatusService.java b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultSystemStatusService.java index 31e529a1b..acc77e4d8 100644 --- a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultSystemStatusService.java +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultSystemStatusService.java @@ -18,8 +18,12 @@ package org.apache.archiva.rest.services; * under the License. */ +import org.apache.archiva.repository.scanner.RepositoryScanner; +import org.apache.archiva.repository.scanner.RepositoryScannerInstance; import org.apache.archiva.rest.api.model.CacheEntry; +import org.apache.archiva.rest.api.model.ConsumerScanningStatistics; import org.apache.archiva.rest.api.model.QueueEntry; +import org.apache.archiva.rest.api.model.RepositoryScannerStatistics; import org.apache.archiva.rest.api.services.ArchivaRestServiceException; import org.apache.archiva.rest.api.services.SystemStatusService; import org.codehaus.plexus.cache.Cache; @@ -34,10 +38,12 @@ import javax.ws.rs.core.Response; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; /** * @author Olivier Lamy @@ -49,17 +55,18 @@ public class DefaultSystemStatusService implements SystemStatusService { - private ApplicationContext applicationContext; private Map queues = null; - Map caches = null; + private Map caches = null; + + private RepositoryScanner scanner; @Inject - public DefaultSystemStatusService( ApplicationContext applicationContext ) + public DefaultSystemStatusService( ApplicationContext applicationContext, RepositoryScanner scanner ) { - this.applicationContext = applicationContext; + this.scanner = scanner; queues = getBeansOfType( applicationContext, TaskQueue.class ); @@ -150,4 +157,48 @@ public class DefaultSystemStatusService } return Boolean.TRUE; } + + public List getRepositoryScannerStatistics() + throws ArchivaRestServiceException + { + Set repositoryScannerInstances = scanner.getInProgressScans(); + if ( repositoryScannerInstances.isEmpty() ) + { + return Collections.emptyList(); + } + List repositoryScannerStatisticsList = + new ArrayList( repositoryScannerInstances.size() ); + + for ( RepositoryScannerInstance instance : repositoryScannerInstances ) + { + RepositoryScannerStatistics repositoryScannerStatistics = new RepositoryScannerStatistics(); + repositoryScannerStatisticsList.add( repositoryScannerStatistics ); + repositoryScannerStatistics.setManagedRepository( instance.getRepository() ); + //repositoryScannerStatistics.setRepositoryScanStatistics( instance.getStatistics() ); + //repositoryScannerStatistics.setConsumerCounts( new HashMap( instance.getConsumerCounts() ) ); + //repositoryScannerStatistics.setConsumerTimings( + // new HashMap( instance.getConsumerTimings() ) ); + repositoryScannerStatistics.setNewFileCount( instance.getStats().getNewFileCount() ); + repositoryScannerStatistics.setTotalFileCount( instance.getStats().getTotalFileCount() ); + repositoryScannerStatistics.setConsumerScanningStatistics( mapConsumerScanningStatistics( instance ) ); + } + + return repositoryScannerStatisticsList; + } + + private List mapConsumerScanningStatistics( RepositoryScannerInstance instance ) + { + // FIXME take care of NPE here !!! + List ret = + new ArrayList( instance.getConsumerCounts().size() ); + for ( Map.Entry entry : instance.getConsumerCounts().entrySet() ) + { + ConsumerScanningStatistics consumerScanningStatistics = new ConsumerScanningStatistics(); + consumerScanningStatistics.setConsumerKey( entry.getKey() ); + consumerScanningStatistics.setCount( entry.getValue() ); + consumerScanningStatistics.setTime( instance.getConsumerTimings().get( entry.getKey() ) ); + ret.add( consumerScanningStatistics ); + } + return ret; + } } diff --git a/archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/general-admin.js b/archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/general-admin.js index e7933568f..c53ee3f55 100644 --- a/archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/general-admin.js +++ b/archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/general-admin.js @@ -627,6 +627,34 @@ $(function() { }); } + mapRepositoryScannerStatisticsList=function(data){ + if(data!=null){ + return $.isArray(data)? $.map(data,function(item){ + return mapRepositoryScannerStatistics(item); + }):[data]; + } + return []; + } + + + mapRepositoryScannerStatistics=function(data){ + return new RepositoryScannerStatistics(mapManagedRepository(data.managedRepository),data.totalFileCount, + data.newFileCount,data.consumerScanningStatistics); + } + + RepositoryScannerStatistics=function(managedRepository,totalFileCount,newFileCount,consumerScanningStatistics){ + //private ManagedRepository managedRepository; + this.managedRepository=managedRepository + + this.consumerScanningStatistics= consumerScanningStatistics; + + //private long totalFileCount = 0; + this.totalFileCount=totalFileCount; + + //private long newFileCount = 0; + this.newFileCount=newFileCount; + } + displaySystemStatus=function(){ screenChange(); var mainContent=$("#main-content"); @@ -663,6 +691,18 @@ $(function() { } }); + var dataStr='[{"managedRepository":{"id":"snapshots","name":"Archiva Managed Snapshot Repository","layout":"default","indexDirectory":null,"location":"/Users/olamy/dev/tests/archiva-appserver-base-test/data/repositories/snapshots","snapshots":true,"releases":false,"blockRedeployments":false,"cronExpression":"0 0,30 * * * ?","stagingRepository":null,"scanned":true,"daysOlder":30,"retentionCount":2,"deleteReleasedSnapshots":false,"stageRepoNeeded":false,"resetStats":false},"consumerCounts":{"create-missing-checksums":114,"duplicate-artifacts":12,"metadata-updater":114,"index-content":197,"create-archiva-metadata":113},"consumerTimings":{"create-missing-checksums":86,"duplicate-artifacts":929,"metadata-updater":263,"index-content":13,"create-archiva-metadata":11088},"totalFileCount":592,"newFileCount":592},{"managedRepository":{"id":"internal","name":"the Archiva Managed Internal Repository","layout":"default","indexDirectory":null,"location":"/Users/olamy/dev/tests/archiva-appserver-base-test/data/repositories/internal","snapshots":false,"releases":true,"blockRedeployments":true,"cronExpression":"0 */5 * * * ?","stagingRepository":null,"scanned":true,"daysOlder":30,"retentionCount":2,"deleteReleasedSnapshots":false,"stageRepoNeeded":false,"resetStats":false},"consumerCounts":{"create-missing-checksums":28,"duplicate-artifacts":28,"metadata-updater":28,"index-content":52,"create-archiva-metadata":28},"consumerTimings":{"create-missing-checksums":71,"duplicate-artifacts":4151,"metadata-updater":2645,"index-content":1,"create-archiva-metadata":3971},"totalFileCount":157,"newFileCount":157}]'; + var data= mapRepositoryScannerStatisticsList( $.parseJSON(dataStr)); + $.log("size:"+data.length); + mainContent.find("#status_scanning" ).html($("#status_scanning_tmpl").tmpl({repositoryScannerStatisticsList:data})); + + $.ajax("restServices/archivaServices/systemStatusService/repositoryScannerStatistics", { + type: "GET", + success: function(data){ + //mainContent.find("#status_scanning" ).html("#status_scanning_tmpl" ).tmpl(data); + } + }); + displayCacheEntries(); } diff --git a/archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/templates/general-admin.html b/archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/templates/general-admin.html index 28ea2acb3..a0a6edcc7 100644 --- a/archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/templates/general-admin.html +++ b/archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/templates/general-admin.html @@ -318,14 +318,14 @@

${$.i18n.prop('system-status.header.queues')}

- +

${$.i18n.prop('system-status.header.scanning')}

- +
@@ -416,5 +416,30 @@ + + -- 2.39.5