From 4d44cf806ddfa8d051f2d6b1289fa3b67b0daf2e Mon Sep 17 00:00:00 2001 From: James Moger Date: Fri, 21 Oct 2011 17:00:07 -0400 Subject: [PATCH] Cache repository sizes and default metrics for performance boost --- docs/00_index.mkd | 2 + docs/04_releases.mkd | 2 + src/com/gitblit/GitBlit.java | 56 +++++++++++- src/com/gitblit/models/ObjectCache.java | 85 +++++++++++++++++++ src/com/gitblit/wicket/pages/SummaryPage.java | 3 +- 5 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 src/com/gitblit/models/ObjectCache.java diff --git a/docs/00_index.mkd b/docs/00_index.mkd index ee5faafb..6a2d0d18 100644 --- a/docs/00_index.mkd +++ b/docs/00_index.mkd @@ -31,6 +31,8 @@ Gitblit requires a Java 6 Runtime Environment (JRE) or a Java 6 Development Kit - improved: updated ui with Twitter's Bootstrap CSS toolkit **New:** *web.loginMessage = gitblit* +- improved: repositories list performance by caching repository sizes (issue 27) +- improved: summary page performance by caching metric calculations (issue 25) - added: authenticated JSON RPC mechanism **New:** *web.enableRpcServlet = true* **New:** *web.enableRpcAdministration = false* diff --git a/docs/04_releases.mkd b/docs/04_releases.mkd index fad61d32..ba09075f 100644 --- a/docs/04_releases.mkd +++ b/docs/04_releases.mkd @@ -5,6 +5,8 @@ - improved: updated ui with Twitter's Bootstrap CSS toolkit **New:** *web.loginMessage = gitblit* +- improved: repositories list performance by caching repository sizes (issue 27) +- improved: summary page performance by caching metric calculations (issue 25) - added: authenticated JSON RPC mechanism **New:** *web.enableRpcServlet = true* **New:** *web.enableRpcAdministration = false* diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java index 50eeb9eb..51d5612e 100644 --- a/src/com/gitblit/GitBlit.java +++ b/src/com/gitblit/GitBlit.java @@ -60,12 +60,15 @@ import com.gitblit.Constants.FederationToken; import com.gitblit.models.FederationModel; import com.gitblit.models.FederationProposal; import com.gitblit.models.FederationSet; +import com.gitblit.models.Metric; +import com.gitblit.models.ObjectCache; import com.gitblit.models.RepositoryModel; import com.gitblit.models.UserModel; import com.gitblit.utils.ByteFormat; import com.gitblit.utils.FederationUtils; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.JsonUtils; +import com.gitblit.utils.MetricUtils; import com.gitblit.utils.StringUtils; /** @@ -97,6 +100,10 @@ public class GitBlit implements ServletContextListener { private final Map federationPullResults = new ConcurrentHashMap(); + private final ObjectCache repositorySizeCache = new ObjectCache(); + + private final ObjectCache> repositoryMetricsCache = new ObjectCache>(); + private RepositoryResolver repositoryResolver; private File repositoriesFolder; @@ -418,6 +425,16 @@ public class GitBlit implements ServletContextListener { } } + /** + * Clears all the cached data for the specified repository. + * + * @param repositoryName + */ + public void clearRepositoryCache(String repositoryName) { + repositorySizeCache.remove(repositoryName); + repositoryMetricsCache.remove(repositoryName); + } + /** * Returns the list of all repositories available to Gitblit. This method * does not consider user access permissions. @@ -550,14 +567,22 @@ public class GitBlit implements ServletContextListener { } /** - * Returns the size in bytes of the repository. + * Returns the size in bytes of the repository. Gitblit caches the + * repository sizes to reduce the performance penalty of recursive + * calculation. The cache is updated if the repository has been changed + * since the last calculation. * * @param model * @return size in bytes */ public long calculateSize(RepositoryModel model) { + if (repositorySizeCache.hasCurrent(model.name, model.lastChange)) { + return repositorySizeCache.getObject(model.name); + } File gitDir = FileKey.resolve(new File(repositoriesFolder, model.name), FS.DETECTED); - return com.gitblit.utils.FileUtils.folderSize(gitDir); + long size = com.gitblit.utils.FileUtils.folderSize(gitDir); + repositorySizeCache.updateObject(model.name, model.lastChange, size); + return size; } /** @@ -596,7 +621,26 @@ public class GitBlit implements ServletContextListener { } /** - * Returns the gitblit string vlaue for the specified key. If key is not + * Returns the metrics for the default branch of the specified repository. + * This method builds a metrics cache. The cache is updated if the + * repository is updated. A new copy of the metrics list is returned on each + * call so that modifications to the list are non-destructive. + * + * @param model + * @param repository + * @return a new array list of metrics + */ + public List getRepositoryDefaultMetrics(RepositoryModel model, Repository repository) { + if (repositoryMetricsCache.hasCurrent(model.name, model.lastChange)) { + return new ArrayList(repositoryMetricsCache.getObject(model.name)); + } + List metrics = MetricUtils.getDateMetrics(repository, null, true, null); + repositoryMetricsCache.updateObject(model.name, model.lastChange, metrics); + return new ArrayList(metrics); + } + + /** + * Returns the gitblit string value for the specified key. If key is not * set, returns defaultValue. * * @param config @@ -678,6 +722,9 @@ public class GitBlit implements ServletContextListener { "Failed to rename repository permissions ''{0}'' to ''{1}''.", repositoryName, repository.name)); } + + // clear the cache + clearRepositoryCache(repositoryName); } // load repository @@ -758,6 +805,9 @@ public class GitBlit implements ServletContextListener { return true; } } + + // clear the repository cache + clearRepositoryCache(repositoryName); } catch (Throwable t) { logger.error(MessageFormat.format("Failed to delete repository {0}", repositoryName), t); } diff --git a/src/com/gitblit/models/ObjectCache.java b/src/com/gitblit/models/ObjectCache.java new file mode 100644 index 00000000..48ede90e --- /dev/null +++ b/src/com/gitblit/models/ObjectCache.java @@ -0,0 +1,85 @@ +/* + * Copyright 2011 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.models; + +import java.io.Serializable; +import java.util.Date; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Reusable object cache. + * + * @author James Moger + * + */ +public class ObjectCache implements Serializable { + + private static final long serialVersionUID = 1L; + + private final Map> cache = new ConcurrentHashMap>(); + + private class CachedObject { + + public final String name; + + private volatile Date date; + + private volatile Y object; + + CachedObject(String name) { + this.name = name; + date = new Date(0); + } + + @Override + public String toString() { + return getClass().getSimpleName() + ": " + name; + } + } + + public boolean hasCurrent(String name, Date date) { + return cache.containsKey(name) && cache.get(name).date.compareTo(date) == 0; + } + + public Date getDate(String name) { + return cache.get(name).date; + } + + public X getObject(String name) { + return cache.get(name).object; + } + + public void updateObject(String name, X object) { + this.updateObject(name, new Date(), object); + } + + public void updateObject(String name, Date date, X object) { + CachedObject obj; + if (cache.containsKey(name)) { + obj = cache.get(name); + } else { + obj = new CachedObject(name); + cache.put(name, obj); + } + obj.date = date; + obj.object = object; + } + + public Object remove(String name) { + return cache.remove(name).object; + } +} diff --git a/src/com/gitblit/wicket/pages/SummaryPage.java b/src/com/gitblit/wicket/pages/SummaryPage.java index 9fd90c5f..50810a18 100644 --- a/src/com/gitblit/wicket/pages/SummaryPage.java +++ b/src/com/gitblit/wicket/pages/SummaryPage.java @@ -45,7 +45,6 @@ import com.gitblit.models.Metric; import com.gitblit.models.PathModel; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.MarkdownUtils; -import com.gitblit.utils.MetricUtils; import com.gitblit.utils.StringUtils; import com.gitblit.utils.TimeUtils; import com.gitblit.wicket.WicketUtils; @@ -68,7 +67,7 @@ public class SummaryPage extends RepositoryPage { List metrics = null; Metric metricsTotal = null; if (GitBlit.getBoolean(Keys.web.generateActivityGraph, true)) { - metrics = MetricUtils.getDateMetrics(r, null, true, null); + metrics = GitBlit.self().getRepositoryDefaultMetrics(getRepositoryModel(), r); metricsTotal = metrics.remove(0); } -- 2.39.5