Cache repository sizes and default metrics for performance boost

docs/00_index.mkd

@@ -31,6 +31,8 @@ Gitblit requires a Java 6 Runtime Environment (JRE) or a Java 6 Development Kit
 
 - improved: updated ui with Twitter's Bootstrap CSS toolkit  
     **New:** *web.loginMessage = gitblit*
+- improved: repositories list performance by caching repository sizes (issue 27)
+- improved: summary page performance by caching metric calculations (issue 25)
 - added: authenticated JSON RPC mechanism   
     **New:** *web.enableRpcServlet = true*  
     **New:** *web.enableRpcAdministration = false*

docs/04_releases.mkd

@@ -5,6 +5,8 @@
 
 - improved: updated ui with Twitter's Bootstrap CSS toolkit  
     **New:** *web.loginMessage = gitblit*
+- improved: repositories list performance by caching repository sizes (issue 27)
+- improved: summary page performance by caching metric calculations (issue 25)
 - added: authenticated JSON RPC mechanism   
     **New:** *web.enableRpcServlet = true*  
     **New:** *web.enableRpcAdministration = false*

src/com/gitblit/GitBlit.java

@@ -60,12 +60,15 @@ import com.gitblit.Constants.FederationToken;
 import com.gitblit.models.FederationModel;
 import com.gitblit.models.FederationProposal;
 import com.gitblit.models.FederationSet;
+import com.gitblit.models.Metric;
+import com.gitblit.models.ObjectCache;
 import com.gitblit.models.RepositoryModel;
 import com.gitblit.models.UserModel;
 import com.gitblit.utils.ByteFormat;
 import com.gitblit.utils.FederationUtils;
 import com.gitblit.utils.JGitUtils;
 import com.gitblit.utils.JsonUtils;
+import com.gitblit.utils.MetricUtils;
 import com.gitblit.utils.StringUtils;
 
 /**
@@ -97,6 +100,10 @@ public class GitBlit implements ServletContextListener {
 
 	private final Map<String, FederationModel> federationPullResults = new ConcurrentHashMap<String, FederationModel>();
 
+	private final ObjectCache<Long> repositorySizeCache = new ObjectCache<Long>();
+
+	private final ObjectCache<List<Metric>> repositoryMetricsCache = new ObjectCache<List<Metric>>();
+
 	private RepositoryResolver<Void> repositoryResolver;
 
 	private File repositoriesFolder;
@@ -418,6 +425,16 @@ public class GitBlit implements ServletContextListener {
 		}
 	}
 
+	/**
+	 * Clears all the cached data for the specified repository.
+	 * 
+	 * @param repositoryName
+	 */
+	public void clearRepositoryCache(String repositoryName) {
+		repositorySizeCache.remove(repositoryName);
+		repositoryMetricsCache.remove(repositoryName);
+	}
+
 	/**
 	 * Returns the list of all repositories available to Gitblit. This method
 	 * does not consider user access permissions.
@@ -550,14 +567,22 @@ public class GitBlit implements ServletContextListener {
 	}
 
 	/**
-	 * Returns the size in bytes of the repository.
+	 * Returns the size in bytes of the repository. Gitblit caches the
+	 * repository sizes to reduce the performance penalty of recursive
+	 * calculation. The cache is updated if the repository has been changed
+	 * since the last calculation.
 	 * 
 	 * @param model
 	 * @return size in bytes
 	 */
 	public long calculateSize(RepositoryModel model) {
+		if (repositorySizeCache.hasCurrent(model.name, model.lastChange)) {
+			return repositorySizeCache.getObject(model.name);
+		}
 		File gitDir = FileKey.resolve(new File(repositoriesFolder, model.name), FS.DETECTED);
-		return com.gitblit.utils.FileUtils.folderSize(gitDir);
+		long size = com.gitblit.utils.FileUtils.folderSize(gitDir);
+		repositorySizeCache.updateObject(model.name, model.lastChange, size);
+		return size;
 	}
 
 	/**
@@ -596,7 +621,26 @@ public class GitBlit implements ServletContextListener {
 	}
 
 	/**
-	 * Returns the gitblit string vlaue for the specified key. If key is not
+	 * Returns the metrics for the default branch of the specified repository.
+	 * This method builds a metrics cache. The cache is updated if the
+	 * repository is updated. A new copy of the metrics list is returned on each
+	 * call so that modifications to the list are non-destructive.
+	 * 
+	 * @param model
+	 * @param repository
+	 * @return a new array list of metrics
+	 */
+	public List<Metric> getRepositoryDefaultMetrics(RepositoryModel model, Repository repository) {
+		if (repositoryMetricsCache.hasCurrent(model.name, model.lastChange)) {
+			return new ArrayList<Metric>(repositoryMetricsCache.getObject(model.name));
+		}
+		List<Metric> metrics = MetricUtils.getDateMetrics(repository, null, true, null);
+		repositoryMetricsCache.updateObject(model.name, model.lastChange, metrics);
+		return new ArrayList<Metric>(metrics);
+	}
+
+	/**
+	 * Returns the gitblit string value for the specified key. If key is not
 	 * set, returns defaultValue.
 	 * 
 	 * @param config
@@ -678,6 +722,9 @@ public class GitBlit implements ServletContextListener {
 							"Failed to rename repository permissions ''{0}'' to ''{1}''.",
 							repositoryName, repository.name));
 				}
+
+				// clear the cache
+				clearRepositoryCache(repositoryName);
 			}
 
 			// load repository
@@ -758,6 +805,9 @@ public class GitBlit implements ServletContextListener {
 					return true;
 				}
 			}
+
+			// clear the repository cache
+			clearRepositoryCache(repositoryName);
 		} catch (Throwable t) {
 			logger.error(MessageFormat.format("Failed to delete repository {0}", repositoryName), t);
 		}

src/com/gitblit/models/ObjectCache.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2011 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.models;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Reusable object cache.
+ * 
+ * @author James Moger
+ * 
+ */
+public class ObjectCache<X> implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private final Map<String, CachedObject<X>> cache = new ConcurrentHashMap<String, CachedObject<X>>();
+
+	private class CachedObject<Y> {
+
+		public final String name;
+
+		private volatile Date date;
+
+		private volatile Y object;
+
+		CachedObject(String name) {
+			this.name = name;
+			date = new Date(0);
+		}
+
+		@Override
+		public String toString() {
+			return getClass().getSimpleName() + ": " + name;
+		}
+	}
+
+	public boolean hasCurrent(String name, Date date) {
+		return cache.containsKey(name) && cache.get(name).date.compareTo(date) == 0;
+	}
+
+	public Date getDate(String name) {
+		return cache.get(name).date;
+	}
+
+	public X getObject(String name) {
+		return cache.get(name).object;
+	}
+
+	public void updateObject(String name, X object) {
+		this.updateObject(name, new Date(), object);
+	}
+
+	public void updateObject(String name, Date date, X object) {
+		CachedObject<X> obj;
+		if (cache.containsKey(name)) {
+			obj = cache.get(name);
+		} else {
+			obj = new CachedObject<X>(name);
+			cache.put(name, obj);
+		}
+		obj.date = date;
+		obj.object = object;
+	}
+
+	public Object remove(String name) {
+		return cache.remove(name).object;
+	}
+}

src/com/gitblit/wicket/pages/SummaryPage.java

@@ -45,7 +45,6 @@ import com.gitblit.models.Metric;
 import com.gitblit.models.PathModel;
 import com.gitblit.utils.JGitUtils;
 import com.gitblit.utils.MarkdownUtils;
-import com.gitblit.utils.MetricUtils;
 import com.gitblit.utils.StringUtils;
 import com.gitblit.utils.TimeUtils;
 import com.gitblit.wicket.WicketUtils;
@@ -68,7 +67,7 @@ public class SummaryPage extends RepositoryPage {
 		List<Metric> metrics = null;
 		Metric metricsTotal = null;
 		if (GitBlit.getBoolean(Keys.web.generateActivityGraph, true)) {
-			metrics = MetricUtils.getDateMetrics(r, null, true, null);
+			metrics = GitBlit.self().getRepositoryDefaultMetrics(getRepositoryModel(), r);
 			metricsTotal = metrics.remove(0);
 		}