Cache repository models to improve performance (issue 103)

1 files changed, 113 insertions, 24 deletions

diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java
index 0940c768..f8fbfefd 100644
--- a/src/com/gitblit/GitBlit.java
+++ b/src/com/gitblit/GitBlit.java
@@ -39,7 +39,6 @@ import java.util.Set;
 import java.util.TimeZone;

 import java.util.TreeSet;

 import java.util.concurrent.ConcurrentHashMap;

-import java.util.concurrent.CopyOnWriteArrayList;

 import java.util.concurrent.Executors;

 import java.util.concurrent.ScheduledExecutorService;

 import java.util.concurrent.TimeUnit;

@@ -58,6 +57,7 @@ import org.eclipse.jgit.errors.RepositoryNotFoundException;
 import org.eclipse.jgit.lib.Repository;

 import org.eclipse.jgit.lib.RepositoryCache.FileKey;

 import org.eclipse.jgit.lib.StoredConfig;

+import org.eclipse.jgit.storage.file.FileBasedConfig;

 import org.eclipse.jgit.storage.file.WindowCache;

 import org.eclipse.jgit.storage.file.WindowCacheConfig;

 import org.eclipse.jgit.transport.ServiceMayNotContinueException;

@@ -88,6 +88,7 @@ import com.gitblit.models.TeamModel;
 import com.gitblit.models.UserModel;

 import com.gitblit.utils.ArrayUtils;

 import com.gitblit.utils.ByteFormat;

+import com.gitblit.utils.DeepCopier;

 import com.gitblit.utils.FederationUtils;

 import com.gitblit.utils.JGitUtils;

 import com.gitblit.utils.JsonUtils;

@@ -128,7 +129,7 @@ public class GitBlit implements ServletContextListener {
 

 	private final ObjectCache<List<Metric>> repositoryMetricsCache = new ObjectCache<List<Metric>>();

 	

-	private final List<String> repositoryListCache = new CopyOnWriteArrayList<String>();

+	private final Map<String, RepositoryModel> repositoryListCache = new ConcurrentHashMap<String, RepositoryModel>();

 	

 	private final AtomicReference<String> repositoryListSettingsChecksum = new AtomicReference<String>("");

 

@@ -736,25 +737,32 @@ public class GitBlit implements ServletContextListener {
 	 * 

 	 * @param name

 	 */

-	private void addToCachedRepositoryList(String name) {

+	private void addToCachedRepositoryList(String name, RepositoryModel model) {

 		if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) {

-			repositoryListCache.add(name);

+			repositoryListCache.put(name, model);

 		}

 	}

+	

+	/**

+	 * Removes the repository from the list of cached repositories.

+	 * 

+	 * @param name

+	 */

+	private void removeFromCachedRepositoryList(String name) {

+		if (StringUtils.isEmpty(name)) {

+			return;

+		}

+		repositoryListCache.remove(name);

+	}

 

 	/**

-	 * Clears all the cached data for the specified repository.

+	 * Clears all the cached metadata for the specified repository.

 	 * 

 	 * @param repositoryName

-	 * @param isDeleted

 	 */

-	private void clearRepositoryCache(String repositoryName, boolean isDeleted) {

+	private void clearRepositoryMetadataCache(String repositoryName) {

 		repositorySizeCache.remove(repositoryName);

 		repositoryMetricsCache.remove(repositoryName);

-		

-		if (isDeleted) {

-			repositoryListCache.remove(repositoryName);

-		}

 	}

 	

 	/**

@@ -831,18 +839,20 @@ public class GitBlit implements ServletContextListener {
 							calculateSize(model);

 						}

 					}

+				} else {

+					// update cache

+					for (String repository : repositories) {

+						getRepositoryModel(repository);

+					}

 				}

 				

-				// update cache

-				repositoryListCache.addAll(repositories);

-				

 				long duration = System.currentTimeMillis() - startTime;

 				logger.info(MessageFormat.format(msg, repositoryListCache.size(), duration));

 			}

 		}

 		

 		// return sorted copy of cached list

-		List<String> list = new ArrayList<String>(repositoryListCache);		

+		List<String> list = new ArrayList<String>(repositoryListCache.keySet());		

 		StringUtils.sortRepositorynames(list);

 		return list;

 	}

@@ -966,6 +976,79 @@ public class GitBlit implements ServletContextListener {
 	 * @return repository model or null

 	 */

 	public RepositoryModel getRepositoryModel(String repositoryName) {

+		if (!repositoryListCache.containsKey(repositoryName)) {

+			RepositoryModel model = loadRepositoryModel(repositoryName);

+			if (model == null) {

+				return null;

+			}

+			addToCachedRepositoryList(repositoryName, model);			

+			return model;

+		}

+		

+		// cached model

+		RepositoryModel model = repositoryListCache.get(repositoryName);

+		

+		// check for updates

+		Repository r = getRepository(repositoryName);

+		if (r == null) {

+			// repository is missing

+			removeFromCachedRepositoryList(repositoryName);

+			logger.error(MessageFormat.format("Repository \"{0}\" is missing! Removing from cache.", repositoryName));

+			return null;

+		}

+		

+		FileBasedConfig config = (FileBasedConfig) getRepositoryConfig(r);

+		if (config.isOutdated()) {

+			// reload model

+			logger.info(MessageFormat.format("Config for \"{0}\" has changed. Reloading model and updating cache.", repositoryName));

+			model = loadRepositoryModel(repositoryName);

+			removeFromCachedRepositoryList(repositoryName);

+			addToCachedRepositoryList(repositoryName, model);

+		} else {

+			// update a few repository parameters 

+			if (!model.hasCommits) {

+				// update hasCommits, assume a repository only gains commits :)

+				model.hasCommits = JGitUtils.hasCommits(r);

+			}

+

+			model.lastChange = JGitUtils.getLastChange(r);

+		}

+		r.close();

+		

+		// return a copy of the cached model

+		return DeepCopier.copy(model);

+	}

+	

+	/**

+	 * Workaround JGit.  I need to access the raw config object directly in order

+	 * to see if the config is dirty so that I can reload a repository model.

+	 * If I use the stock JGit method to get the config it already reloads the

+	 * config.  If the config changes are made within Gitblit this is fine as

+	 * the returned config will still be flagged as dirty.  BUT... if the config

+	 * is manipulated outside Gitblit then it fails to recognize this as dirty.

+	 *  

+	 * @param r

+	 * @return a config

+	 */

+	private StoredConfig getRepositoryConfig(Repository r) {

+		try {

+			Field f = r.getClass().getDeclaredField("repoConfig");

+			f.setAccessible(true);

+			StoredConfig config = (StoredConfig) f.get(r);

+			return config;

+		} catch (Exception e) {

+			logger.error("Failed to retrieve \"repoConfig\" via reflection", e);

+		}

+		return r.getConfig();

+	}

+	

+	/**

+	 * Create a repository model from the configuration and repository data.

+	 * 

+	 * @param repositoryName

+	 * @return a repositoryModel or null if the repository does not exist

+	 */

+	private RepositoryModel loadRepositoryModel(String repositoryName) {

 		Repository r = getRepository(repositoryName);

 		if (r == null) {

 			return null;

@@ -1025,6 +1108,11 @@ public class GitBlit implements ServletContextListener {
 	 * @return true if the repository exists

 	 */

 	public boolean hasRepository(String repositoryName) {

+		if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) {

+			// if we are caching use the cache to determine availability

+			// otherwise we end up adding a phantom repository to the cache

+			return repositoryListCache.containsKey(repositoryName);

+		}		

 		Repository r = getRepository(repositoryName, false);

 		if (r == null) {

 			return false;

@@ -1169,9 +1257,6 @@ public class GitBlit implements ServletContextListener {
 			// create repository

 			logger.info("create repository " + repository.name);

 			r = JGitUtils.createRepository(repositoriesFolder, repository.name);

-			

-			// add name to cache

-			addToCachedRepositoryList(repository.name);

 		} else {

 			// rename repository

 			if (!repositoryName.equalsIgnoreCase(repository.name)) {

@@ -1211,10 +1296,7 @@ public class GitBlit implements ServletContextListener {
 				}

 

 				// clear the cache

-				clearRepositoryCache(repositoryName, true);

-				

-				// add new name to repository list cache

-				addToCachedRepositoryList(repository.name);

+				clearRepositoryMetadataCache(repositoryName);

 			}

 

 			// load repository

@@ -1242,13 +1324,18 @@ public class GitBlit implements ServletContextListener {
 						repository.name, currentRef, repository.HEAD));

 				if (JGitUtils.setHEADtoRef(r, repository.HEAD)) {

 					// clear the cache

-					clearRepositoryCache(repository.name, false);

+					clearRepositoryMetadataCache(repository.name);

 				}

 			}

 

 			// close the repository object

 			r.close();

 		}

+		

+		// update repository cache

+		removeFromCachedRepositoryList(repositoryName);

+		// model will actually be replaced on next load because config is stale

+		addToCachedRepositoryList(repository.name, repository);

 	}

 	

 	/**

@@ -1339,12 +1426,14 @@ public class GitBlit implements ServletContextListener {
 		try {

 			closeRepository(repositoryName);

 			// clear the repository cache

-			clearRepositoryCache(repositoryName, true);			

+			clearRepositoryMetadataCache(repositoryName);

+			removeFromCachedRepositoryList(repositoryName);

 

 			File folder = new File(repositoriesFolder, repositoryName);

 			if (folder.exists() && folder.isDirectory()) {

 				FileUtils.delete(folder, FileUtils.RECURSIVE | FileUtils.RETRY);

 				if (userService.deleteRepositoryRole(repositoryName)) {

+					logger.info(MessageFormat.format("Repository \"{0}\" deleted", repositoryName));

 					return true;

 				}

 			}