Improve performance of cache filling

diff --git a/src/main/java/com/gitblit/GitBlit.java b/src/main/java/com/gitblit/GitBlit.java
index 6fd168a68fed9ce58475f567f72de07e6d84e996..eabaa3f2d01e7796de2ea764bdc2455c7f98e3eb 100644 (file)
--- a/src/main/java/com/gitblit/GitBlit.java
+++ b/src/main/java/com/gitblit/GitBlit.java
@@ -102,6 +102,7 @@ import com.gitblit.models.ForkModel;
 import com.gitblit.models.GitClientApplication;
 import com.gitblit.models.Metric;
 import com.gitblit.models.ProjectModel;
+import com.gitblit.models.RefModel;
 import com.gitblit.models.RegistrantAccessPermission;
 import com.gitblit.models.RepositoryModel;
 import com.gitblit.models.RepositoryUrl;
@@ -3402,9 +3403,8 @@ public class GitBlit implements ServletContextListener {
                configureJGit();
                configureFanout();
                configureGitDaemon();
-               
-               CommitCache.instance().setCacheDays(settings.getInteger(Keys.web.activityCacheDays, 14));
-               
+               configureCommitCache();
+
                ContainerUtils.CVE_2007_0450.test();
        }
        
@@ -3514,6 +3514,42 @@ public class GitBlit implements ServletContextListener {
                }
        }
        
+       protected void configureCommitCache() {
+               int daysToCache = settings.getInteger(Keys.web.activityCacheDays, 14);
+               if (daysToCache <= 0) {
+                       logger.info("commit cache disabled");
+               } else {
+                       long start = System.nanoTime();
+                       long repoCount = 0;
+                       long commitCount = 0;
+                       logger.info(MessageFormat.format("preparing {0} day commit cache. please wait...", daysToCache));
+                       CommitCache.instance().setCacheDays(daysToCache);
+                       Date cutoff = CommitCache.instance().getCutoffDate();
+                       for (String repositoryName : getRepositoryList()) {
+                               RepositoryModel model = getRepositoryModel(repositoryName);
+                               if (model.hasCommits && model.lastChange.after(cutoff)) {
+                                       repoCount++;
+                                       Repository repository = getRepository(repositoryName);
+                                       for (RefModel ref : JGitUtils.getLocalBranches(repository, true, -1)) {
+                                               if (!ref.getDate().after(cutoff)) {
+                                                       // branch not recently updated
+                                                       continue;
+                                               }
+                                               List<?> commits = CommitCache.instance().getCommits(repositoryName, repository, ref.getName());
+                                               if (commits.size() > 0) {
+                                                       logger.info(MessageFormat.format("  cached {0} commits for {1}:{2}",
+                                                                       commits.size(), repositoryName, ref.getName()));
+                                                       commitCount += commits.size();
+                                               }
+                                       }
+                                       repository.close();
+                               }
+                       }
+                       logger.info(MessageFormat.format("built {0} day commit cache of {1} commits across {2} repositories in {3} msecs",
+                                       daysToCache, commitCount, repoCount, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
+               }
+       }
+       
        protected final Logger getLogger() {
                return logger;
        }

diff --git a/src/main/java/com/gitblit/utils/ActivityUtils.java b/src/main/java/com/gitblit/utils/ActivityUtils.java
index fa74350e4aec45b0cb2b0cd1728ca6f24488557d..c4e95879231aa6a3f6efa82cd33e5ac6c2183519 100644 (file)
--- a/src/main/java/com/gitblit/utils/ActivityUtils.java
+++ b/src/main/java/com/gitblit/utils/ActivityUtils.java
@@ -105,6 +105,10 @@ public class ActivityUtils {
                                if (StringUtils.isEmpty(objectId)) {\r
                                        for (RefModel local : JGitUtils.getLocalBranches(\r
                                                        repository, true, -1)) {\r
+                                       if (!local.getDate().after(thresholdDate)) {\r
+                                                       // branch not recently updated\r
+                                               continue;\r
+                                       }\r
                                                branches.add(local.getName());\r
                                        }\r
                                } else {\r

diff --git a/src/main/java/com/gitblit/utils/CommitCache.java b/src/main/java/com/gitblit/utils/CommitCache.java
index e188ff9aab139eb48f7f872cb6e556af79e9e54f..9db5f0c9567c7b41ebe88cbda57f29c27119cc17 100644 (file)
--- a/src/main/java/com/gitblit/utils/CommitCache.java
+++ b/src/main/java/com/gitblit/utils/CommitCache.java
@@ -67,7 +67,7 @@ public class CommitCache {
         * 
         * @return
         */
-       protected Date getCacheCutoffDate() {
+       public Date getCutoffDate() {
                final Calendar cal = Calendar.getInstance();
                cal.setTimeInMillis(System.currentTimeMillis());
                cal.set(Calendar.HOUR_OF_DAY, 0);
@@ -118,7 +118,7 @@ public class CommitCache {
         * @return a list of commits
         */
        public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch) {
-               return getCommits(repositoryName, repository, branch, getCacheCutoffDate());
+               return getCommits(repositoryName, repository, branch, getCutoffDate());
        }
        
        /**
@@ -134,7 +134,7 @@ public class CommitCache {
         */
        public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch, Date sinceDate) {
                long start = System.nanoTime();
-               Date cacheCutoffDate = getCacheCutoffDate();
+               Date cacheCutoffDate = getCutoffDate();
                List<RepositoryCommit> list;
                if (cacheDays > 0 && (sinceDate.getTime() >= cacheCutoffDate.getTime())) {
                        // request fits within the cache window

diff --git a/src/main/java/com/gitblit/utils/RefLogUtils.java b/src/main/java/com/gitblit/utils/RefLogUtils.java
index 73b714686dee0d33f2f8605a73c776cb6727d658..0dd6652441836e24f8e000dd40f0534dc024917b 100644 (file)
--- a/src/main/java/com/gitblit/utils/RefLogUtils.java
+++ b/src/main/java/com/gitblit/utils/RefLogUtils.java
@@ -548,6 +548,10 @@ public class RefLogUtils {
         Map<String, DailyLogEntry> dailydigests = new HashMap<String, DailyLogEntry>();
         String linearParent = null;
         for (RefModel local : JGitUtils.getLocalBranches(repository, true, -1)) {
+               if (!local.getDate().after(minimumDate)) {
+                               // branch not recently updated
+                       continue;
+               }
             String branch = local.getName();
             List<RepositoryCommit> commits = CommitCache.instance().getCommits(repositoryName, repository,  branch, minimumDate);
             for (RepositoryCommit commit : commits) {

diff --git a/src/main/java/com/gitblit/wicket/pages/DashboardPage.java b/src/main/java/com/gitblit/wicket/pages/DashboardPage.java
index db3092ded9a47e6e38eb148173d6262b0cc5238c..68a4279c0c4b21d4d2111e9e369af6e0b2343cd7 100644 (file)
--- a/src/main/java/com/gitblit/wicket/pages/DashboardPage.java
+++ b/src/main/java/com/gitblit/wicket/pages/DashboardPage.java
@@ -16,9 +16,7 @@
 package com.gitblit.wicket.pages;\r
 \r
 import java.io.Serializable;\r
-import java.text.DateFormat;\r
 import java.text.MessageFormat;\r
-import java.text.SimpleDateFormat;\r
 import java.util.ArrayList;\r
 import java.util.Calendar;\r
 import java.util.Collection;\r
@@ -55,7 +53,6 @@ import com.gitblit.wicket.PageRegistration.DropDownMenuRegistration;
 import com.gitblit.wicket.charting.GoogleChart;\r
 import com.gitblit.wicket.charting.GoogleCharts;\r
 import com.gitblit.wicket.charting.GooglePieChart;\r
-import com.gitblit.wicket.ng.NgController;\r
 import com.gitblit.wicket.panels.DigestsPanel;\r
 import com.gitblit.wicket.panels.LinkPanel;\r
 \r
@@ -83,10 +80,15 @@ public abstract class DashboardPage extends RootPage {
                // create daily commit digest feed\r
                List<DailyLogEntry> digests = new ArrayList<DailyLogEntry>();\r
                for (RepositoryModel model : repositories) {\r
-                       Repository repository = GitBlit.self().getRepository(model.name);\r
-                       List<DailyLogEntry> entries = RefLogUtils.getDailyLogByRef(model.name, repository, minimumDate, timezone);\r
-                       digests.addAll(entries);\r
-                       repository.close();\r
+                       if (model.isCollectingGarbage) {\r
+                               continue;\r
+                       }\r
+                       if (model.hasCommits && model.lastChange.after(minimumDate)) {\r
+                               Repository repository = GitBlit.self().getRepository(model.name);\r
+                               List<DailyLogEntry> entries = RefLogUtils.getDailyLogByRef(model.name, repository, minimumDate, timezone);\r
+                               digests.addAll(entries);\r
+                               repository.close();\r
+                       }\r
                }\r
                \r
                Fragment activityFragment = new Fragment("activity", "activityFragment", this);\r