Cache repository sizes and default metrics for performance boost

5 files changed, 143 insertions, 5 deletions

diff --git a/docs/00_index.mkd b/docs/00_index.mkd
index ee5faafb..6a2d0d18 100644
--- a/docs/00_index.mkd
+++ b/docs/00_index.mkd
@@ -31,6 +31,8 @@ Gitblit requires a Java 6 Runtime Environment (JRE) or a Java 6 Development Kit
 

 - improved: updated ui with Twitter's Bootstrap CSS toolkit  

     **New:** *web.loginMessage = gitblit*

+- improved: repositories list performance by caching repository sizes (issue 27)

+- improved: summary page performance by caching metric calculations (issue 25)

 - added: authenticated JSON RPC mechanism   

     **New:** *web.enableRpcServlet = true*  

     **New:** *web.enableRpcAdministration = false*

diff --git a/docs/04_releases.mkd b/docs/04_releases.mkd
index fad61d32..ba09075f 100644
--- a/docs/04_releases.mkd
+++ b/docs/04_releases.mkd
@@ -5,6 +5,8 @@
 

 - improved: updated ui with Twitter's Bootstrap CSS toolkit  

     **New:** *web.loginMessage = gitblit*

+- improved: repositories list performance by caching repository sizes (issue 27)

+- improved: summary page performance by caching metric calculations (issue 25)

 - added: authenticated JSON RPC mechanism   

     **New:** *web.enableRpcServlet = true*  

     **New:** *web.enableRpcAdministration = false*

diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java
index 50eeb9eb..51d5612e 100644
--- a/src/com/gitblit/GitBlit.java
+++ b/src/com/gitblit/GitBlit.java
@@ -60,12 +60,15 @@ import com.gitblit.Constants.FederationToken;
 import com.gitblit.models.FederationModel;

 import com.gitblit.models.FederationProposal;

 import com.gitblit.models.FederationSet;

+import com.gitblit.models.Metric;

+import com.gitblit.models.ObjectCache;

 import com.gitblit.models.RepositoryModel;

 import com.gitblit.models.UserModel;

 import com.gitblit.utils.ByteFormat;

 import com.gitblit.utils.FederationUtils;

 import com.gitblit.utils.JGitUtils;

 import com.gitblit.utils.JsonUtils;

+import com.gitblit.utils.MetricUtils;

 import com.gitblit.utils.StringUtils;

 

 /**

@@ -97,6 +100,10 @@ public class GitBlit implements ServletContextListener {
 

 	private final Map<String, FederationModel> federationPullResults = new ConcurrentHashMap<String, FederationModel>();

 

+	private final ObjectCache<Long> repositorySizeCache = new ObjectCache<Long>();

+

+	private final ObjectCache<List<Metric>> repositoryMetricsCache = new ObjectCache<List<Metric>>();

+

 	private RepositoryResolver<Void> repositoryResolver;

 

 	private File repositoriesFolder;

@@ -419,6 +426,16 @@ public class GitBlit implements ServletContextListener {
 	}

 

 	/**

+	 * Clears all the cached data for the specified repository.

+	 * 

+	 * @param repositoryName

+	 */

+	public void clearRepositoryCache(String repositoryName) {

+		repositorySizeCache.remove(repositoryName);

+		repositoryMetricsCache.remove(repositoryName);

+	}

+

+	/**

 	 * Returns the list of all repositories available to Gitblit. This method

 	 * does not consider user access permissions.

 	 * 

@@ -550,14 +567,22 @@ public class GitBlit implements ServletContextListener {
 	}

 

 	/**

-	 * Returns the size in bytes of the repository.

+	 * Returns the size in bytes of the repository. Gitblit caches the

+	 * repository sizes to reduce the performance penalty of recursive

+	 * calculation. The cache is updated if the repository has been changed

+	 * since the last calculation.

 	 * 

 	 * @param model

 	 * @return size in bytes

 	 */

 	public long calculateSize(RepositoryModel model) {

+		if (repositorySizeCache.hasCurrent(model.name, model.lastChange)) {

+			return repositorySizeCache.getObject(model.name);

+		}

 		File gitDir = FileKey.resolve(new File(repositoriesFolder, model.name), FS.DETECTED);

-		return com.gitblit.utils.FileUtils.folderSize(gitDir);

+		long size = com.gitblit.utils.FileUtils.folderSize(gitDir);

+		repositorySizeCache.updateObject(model.name, model.lastChange, size);

+		return size;

 	}

 

 	/**

@@ -596,7 +621,26 @@ public class GitBlit implements ServletContextListener {
 	}

 

 	/**

-	 * Returns the gitblit string vlaue for the specified key. If key is not

+	 * Returns the metrics for the default branch of the specified repository.

+	 * This method builds a metrics cache. The cache is updated if the

+	 * repository is updated. A new copy of the metrics list is returned on each

+	 * call so that modifications to the list are non-destructive.

+	 * 

+	 * @param model

+	 * @param repository

+	 * @return a new array list of metrics

+	 */

+	public List<Metric> getRepositoryDefaultMetrics(RepositoryModel model, Repository repository) {

+		if (repositoryMetricsCache.hasCurrent(model.name, model.lastChange)) {

+			return new ArrayList<Metric>(repositoryMetricsCache.getObject(model.name));

+		}

+		List<Metric> metrics = MetricUtils.getDateMetrics(repository, null, true, null);

+		repositoryMetricsCache.updateObject(model.name, model.lastChange, metrics);

+		return new ArrayList<Metric>(metrics);

+	}

+

+	/**

+	 * Returns the gitblit string value for the specified key. If key is not

 	 * set, returns defaultValue.

 	 * 

 	 * @param config

@@ -678,6 +722,9 @@ public class GitBlit implements ServletContextListener {
 							"Failed to rename repository permissions ''{0}'' to ''{1}''.",

 							repositoryName, repository.name));

 				}

+

+				// clear the cache

+				clearRepositoryCache(repositoryName);

 			}

 

 			// load repository

@@ -758,6 +805,9 @@ public class GitBlit implements ServletContextListener {
 					return true;

 				}

 			}

+

+			// clear the repository cache

+			clearRepositoryCache(repositoryName);

 		} catch (Throwable t) {

 			logger.error(MessageFormat.format("Failed to delete repository {0}", repositoryName), t);

 		}

diff --git a/src/com/gitblit/models/ObjectCache.java b/src/com/gitblit/models/ObjectCache.java
new file mode 100644
index 00000000..48ede90e
--- /dev/null
+++ b/src/com/gitblit/models/ObjectCache.java
@@ -0,0 +1,85 @@
+/*

+ * Copyright 2011 gitblit.com.

+ *

+ * Licensed under the Apache License, Version 2.0 (the "License");

+ * you may not use this file except in compliance with the License.

+ * You may obtain a copy of the License at

+ *

+ *     http://www.apache.org/licenses/LICENSE-2.0

+ *

+ * Unless required by applicable law or agreed to in writing, software

+ * distributed under the License is distributed on an "AS IS" BASIS,

+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+ * See the License for the specific language governing permissions and

+ * limitations under the License.

+ */

+package com.gitblit.models;

+

+import java.io.Serializable;

+import java.util.Date;

+import java.util.Map;

+import java.util.concurrent.ConcurrentHashMap;

+

+/**

+ * Reusable object cache.

+ * 

+ * @author James Moger

+ * 

+ */

+public class ObjectCache<X> implements Serializable {

+

+	private static final long serialVersionUID = 1L;

+

+	private final Map<String, CachedObject<X>> cache = new ConcurrentHashMap<String, CachedObject<X>>();

+

+	private class CachedObject<Y> {

+

+		public final String name;

+

+		private volatile Date date;

+

+		private volatile Y object;

+

+		CachedObject(String name) {

+			this.name = name;

+			date = new Date(0);

+		}

+

+		@Override

+		public String toString() {

+			return getClass().getSimpleName() + ": " + name;

+		}

+	}

+

+	public boolean hasCurrent(String name, Date date) {

+		return cache.containsKey(name) && cache.get(name).date.compareTo(date) == 0;

+	}

+

+	public Date getDate(String name) {

+		return cache.get(name).date;

+	}

+

+	public X getObject(String name) {

+		return cache.get(name).object;

+	}

+

+	public void updateObject(String name, X object) {

+		this.updateObject(name, new Date(), object);

+	}

+

+	public void updateObject(String name, Date date, X object) {

+		CachedObject<X> obj;

+		if (cache.containsKey(name)) {

+			obj = cache.get(name);

+		} else {

+			obj = new CachedObject<X>(name);

+			cache.put(name, obj);

+		}

+		obj.date = date;

+		obj.object = object;

+	}

+

+	public Object remove(String name) {

+		return cache.remove(name).object;

+	}

+}

diff --git a/src/com/gitblit/wicket/pages/SummaryPage.java b/src/com/gitblit/wicket/pages/SummaryPage.java
index 9fd90c5f..50810a18 100644
--- a/src/com/gitblit/wicket/pages/SummaryPage.java
+++ b/src/com/gitblit/wicket/pages/SummaryPage.java
@@ -45,7 +45,6 @@ import com.gitblit.models.Metric;
 import com.gitblit.models.PathModel;

 import com.gitblit.utils.JGitUtils;

 import com.gitblit.utils.MarkdownUtils;

-import com.gitblit.utils.MetricUtils;

 import com.gitblit.utils.StringUtils;

 import com.gitblit.utils.TimeUtils;

 import com.gitblit.wicket.WicketUtils;

@@ -68,7 +67,7 @@ public class SummaryPage extends RepositoryPage {
 		List<Metric> metrics = null;

 		Metric metricsTotal = null;

 		if (GitBlit.getBoolean(Keys.web.generateActivityGraph, true)) {

-			metrics = MetricUtils.getDateMetrics(r, null, true, null);

+			metrics = GitBlit.self().getRepositoryDefaultMetrics(getRepositoryModel(), r);

 			metricsTotal = metrics.remove(0);

 		}