From 3c4ce1d366641158927cee8b4e47191ed681b524 Mon Sep 17 00:00:00 2001 From: James Moger Date: Mon, 24 Jun 2013 14:04:46 -0400 Subject: [PATCH] Improve performance of cache filling --- src/main/java/com/gitblit/GitBlit.java | 42 +++++++++++++++++-- .../java/com/gitblit/utils/ActivityUtils.java | 4 ++ .../java/com/gitblit/utils/CommitCache.java | 6 +-- .../java/com/gitblit/utils/RefLogUtils.java | 4 ++ .../gitblit/wicket/pages/DashboardPage.java | 16 +++---- 5 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/gitblit/GitBlit.java b/src/main/java/com/gitblit/GitBlit.java index 6fd168a6..eabaa3f2 100644 --- a/src/main/java/com/gitblit/GitBlit.java +++ b/src/main/java/com/gitblit/GitBlit.java @@ -102,6 +102,7 @@ import com.gitblit.models.ForkModel; import com.gitblit.models.GitClientApplication; import com.gitblit.models.Metric; import com.gitblit.models.ProjectModel; +import com.gitblit.models.RefModel; import com.gitblit.models.RegistrantAccessPermission; import com.gitblit.models.RepositoryModel; import com.gitblit.models.RepositoryUrl; @@ -3402,9 +3403,8 @@ public class GitBlit implements ServletContextListener { configureJGit(); configureFanout(); configureGitDaemon(); - - CommitCache.instance().setCacheDays(settings.getInteger(Keys.web.activityCacheDays, 14)); - + configureCommitCache(); + ContainerUtils.CVE_2007_0450.test(); } @@ -3514,6 +3514,42 @@ public class GitBlit implements ServletContextListener { } } + protected void configureCommitCache() { + int daysToCache = settings.getInteger(Keys.web.activityCacheDays, 14); + if (daysToCache <= 0) { + logger.info("commit cache disabled"); + } else { + long start = System.nanoTime(); + long repoCount = 0; + long commitCount = 0; + logger.info(MessageFormat.format("preparing {0} day commit cache. please wait...", daysToCache)); + CommitCache.instance().setCacheDays(daysToCache); + Date cutoff = CommitCache.instance().getCutoffDate(); + for (String repositoryName : getRepositoryList()) { + RepositoryModel model = getRepositoryModel(repositoryName); + if (model.hasCommits && model.lastChange.after(cutoff)) { + repoCount++; + Repository repository = getRepository(repositoryName); + for (RefModel ref : JGitUtils.getLocalBranches(repository, true, -1)) { + if (!ref.getDate().after(cutoff)) { + // branch not recently updated + continue; + } + List commits = CommitCache.instance().getCommits(repositoryName, repository, ref.getName()); + if (commits.size() > 0) { + logger.info(MessageFormat.format(" cached {0} commits for {1}:{2}", + commits.size(), repositoryName, ref.getName())); + commitCount += commits.size(); + } + } + repository.close(); + } + } + logger.info(MessageFormat.format("built {0} day commit cache of {1} commits across {2} repositories in {3} msecs", + daysToCache, commitCount, repoCount, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start))); + } + } + protected final Logger getLogger() { return logger; } diff --git a/src/main/java/com/gitblit/utils/ActivityUtils.java b/src/main/java/com/gitblit/utils/ActivityUtils.java index fa74350e..c4e95879 100644 --- a/src/main/java/com/gitblit/utils/ActivityUtils.java +++ b/src/main/java/com/gitblit/utils/ActivityUtils.java @@ -105,6 +105,10 @@ public class ActivityUtils { if (StringUtils.isEmpty(objectId)) { for (RefModel local : JGitUtils.getLocalBranches( repository, true, -1)) { + if (!local.getDate().after(thresholdDate)) { + // branch not recently updated + continue; + } branches.add(local.getName()); } } else { diff --git a/src/main/java/com/gitblit/utils/CommitCache.java b/src/main/java/com/gitblit/utils/CommitCache.java index e188ff9a..9db5f0c9 100644 --- a/src/main/java/com/gitblit/utils/CommitCache.java +++ b/src/main/java/com/gitblit/utils/CommitCache.java @@ -67,7 +67,7 @@ public class CommitCache { * * @return */ - protected Date getCacheCutoffDate() { + public Date getCutoffDate() { final Calendar cal = Calendar.getInstance(); cal.setTimeInMillis(System.currentTimeMillis()); cal.set(Calendar.HOUR_OF_DAY, 0); @@ -118,7 +118,7 @@ public class CommitCache { * @return a list of commits */ public List getCommits(String repositoryName, Repository repository, String branch) { - return getCommits(repositoryName, repository, branch, getCacheCutoffDate()); + return getCommits(repositoryName, repository, branch, getCutoffDate()); } /** @@ -134,7 +134,7 @@ public class CommitCache { */ public List getCommits(String repositoryName, Repository repository, String branch, Date sinceDate) { long start = System.nanoTime(); - Date cacheCutoffDate = getCacheCutoffDate(); + Date cacheCutoffDate = getCutoffDate(); List list; if (cacheDays > 0 && (sinceDate.getTime() >= cacheCutoffDate.getTime())) { // request fits within the cache window diff --git a/src/main/java/com/gitblit/utils/RefLogUtils.java b/src/main/java/com/gitblit/utils/RefLogUtils.java index 73b71468..0dd66524 100644 --- a/src/main/java/com/gitblit/utils/RefLogUtils.java +++ b/src/main/java/com/gitblit/utils/RefLogUtils.java @@ -548,6 +548,10 @@ public class RefLogUtils { Map dailydigests = new HashMap(); String linearParent = null; for (RefModel local : JGitUtils.getLocalBranches(repository, true, -1)) { + if (!local.getDate().after(minimumDate)) { + // branch not recently updated + continue; + } String branch = local.getName(); List commits = CommitCache.instance().getCommits(repositoryName, repository, branch, minimumDate); for (RepositoryCommit commit : commits) { diff --git a/src/main/java/com/gitblit/wicket/pages/DashboardPage.java b/src/main/java/com/gitblit/wicket/pages/DashboardPage.java index db3092de..68a4279c 100644 --- a/src/main/java/com/gitblit/wicket/pages/DashboardPage.java +++ b/src/main/java/com/gitblit/wicket/pages/DashboardPage.java @@ -16,9 +16,7 @@ package com.gitblit.wicket.pages; import java.io.Serializable; -import java.text.DateFormat; import java.text.MessageFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; @@ -55,7 +53,6 @@ import com.gitblit.wicket.PageRegistration.DropDownMenuRegistration; import com.gitblit.wicket.charting.GoogleChart; import com.gitblit.wicket.charting.GoogleCharts; import com.gitblit.wicket.charting.GooglePieChart; -import com.gitblit.wicket.ng.NgController; import com.gitblit.wicket.panels.DigestsPanel; import com.gitblit.wicket.panels.LinkPanel; @@ -83,10 +80,15 @@ public abstract class DashboardPage extends RootPage { // create daily commit digest feed List digests = new ArrayList(); for (RepositoryModel model : repositories) { - Repository repository = GitBlit.self().getRepository(model.name); - List entries = RefLogUtils.getDailyLogByRef(model.name, repository, minimumDate, timezone); - digests.addAll(entries); - repository.close(); + if (model.isCollectingGarbage) { + continue; + } + if (model.hasCommits && model.lastChange.after(minimumDate)) { + Repository repository = GitBlit.self().getRepository(model.name); + List entries = RefLogUtils.getDailyLogByRef(model.name, repository, minimumDate, timezone); + digests.addAll(entries); + repository.close(); + } } Fragment activityFragment = new Fragment("activity", "activityFragment", this); -- 2.39.5