From 79dd0bddbd939c85f002f3a33b95ae84d0bf38cb Mon Sep 17 00:00:00 2001 From: James Moger Date: Mon, 24 Jun 2013 09:36:19 -0400 Subject: Implemented commit cache for the dashboards, activity, and project pages --- releases.moxie | 4 +- src/main/distrib/data/gitblit.properties | 16 +- src/main/java/com/gitblit/GitBlit.java | 3 + src/main/java/com/gitblit/models/Activity.java | 26 ++- src/main/java/com/gitblit/models/RefLogEntry.java | 19 +- .../java/com/gitblit/models/RepositoryCommit.java | 20 +- src/main/java/com/gitblit/utils/ActivityUtils.java | 17 +- src/main/java/com/gitblit/utils/CommitCache.java | 253 +++++++++++++++++++++ src/main/java/com/gitblit/utils/ObjectCache.java | 2 +- src/main/java/com/gitblit/utils/RefLogUtils.java | 14 +- .../com/gitblit/wicket/panels/ActivityPanel.java | 3 +- 11 files changed, 346 insertions(+), 31 deletions(-) create mode 100644 src/main/java/com/gitblit/utils/CommitCache.java diff --git a/releases.moxie b/releases.moxie index 8ac0c491..02f1c7bc 100644 --- a/releases.moxie +++ b/releases.moxie @@ -56,6 +56,7 @@ r17: { - Setting to automatically create an user account based on an authenticated user principal from the servlet container (issue-246) - Added WindowsUserService to authenticate users against Windows accounts (issue-250) - Global and per-repository setting to exclude authors from metrics (issue-251) + - Added commit cache to improve Activity, Dashboard, and Project page generation times - Added SalesForce.com user service - Added simple star/unstar function to flag or bookmark interesting repositories - Added Dashboard page which shows a news feed for starred repositories and offers a filterable list of repositories you care about @@ -126,7 +127,8 @@ r17: { - { name: 'realm.salesforce.orgId', defaultValue: 0 } - { name: 'realm.windows.defaultDomain', defaultValue: ' ' } - { name: 'realm.windows.backingUserService', defaultValue: 'users.conf' } - - { name: 'web.activityDurationChoices', defaultValue: '7 14 28 60 90 180' } + - { name: 'web.activityDurationChoices', defaultValue: '7 14 21 28' } + - { name: 'web.activityCacheDays', defaultValue: 14 } - { name: 'web.allowAppCloneLinks', defaultValue: 'true' } - { name: 'web.forceDefaultLocale', defaultValue: ' ' } - { name: 'web.metricAuthorExclusions', defaultValue: ' ' } diff --git a/src/main/distrib/data/gitblit.properties b/src/main/distrib/data/gitblit.properties index f38afefc..a3270e45 100644 --- a/src/main/distrib/data/gitblit.properties +++ b/src/main/distrib/data/gitblit.properties @@ -825,7 +825,21 @@ web.activityDuration = 7 # # SPACE-DELIMITED # SINCE 1.3.0 -web.activityDurationChoices = 7 14 28 60 90 180 +web.activityDurationChoices = 7 14 21 28 + +# The number of days of commits to cache in memory for the dashboard, activity, +# and project pages. A value of 0 will disable all caching and will parse commits +# in each repository per-request. If the value > 0 these pages will try to fulfill +# requests using the commit cache. If the request specifies a period which falls +# outside the commit cache window, then the cache will be ignored and the request +# will be fulfilled by brute-force parsing all relevant commits per-repository. +# +# Consider the values specified for *web.activityDurationChoices* when setting +# the cache size AND consider adjusting the JVM -Xmx heap parameter appropriately. +# +# SINCE 1.3.0 +# RESTART REQUIRED +web.activityCacheDays = 14 # Case-insensitive list of authors to exclude from metrics. Useful for # eliminating bots. diff --git a/src/main/java/com/gitblit/GitBlit.java b/src/main/java/com/gitblit/GitBlit.java index c6e24d77..6fd168a6 100644 --- a/src/main/java/com/gitblit/GitBlit.java +++ b/src/main/java/com/gitblit/GitBlit.java @@ -114,6 +114,7 @@ import com.gitblit.models.UserModel; import com.gitblit.utils.ArrayUtils; import com.gitblit.utils.Base64; import com.gitblit.utils.ByteFormat; +import com.gitblit.utils.CommitCache; import com.gitblit.utils.ContainerUtils; import com.gitblit.utils.DeepCopier; import com.gitblit.utils.FederationUtils; @@ -3402,6 +3403,8 @@ public class GitBlit implements ServletContextListener { configureFanout(); configureGitDaemon(); + CommitCache.instance().setCacheDays(settings.getInteger(Keys.web.activityCacheDays, 14)); + ContainerUtils.CVE_2007_0450.test(); } diff --git a/src/main/java/com/gitblit/models/Activity.java b/src/main/java/com/gitblit/models/Activity.java index 547c348f..8af86d61 100644 --- a/src/main/java/com/gitblit/models/Activity.java +++ b/src/main/java/com/gitblit/models/Activity.java @@ -103,14 +103,28 @@ public class Activity implements Serializable, Comparable { */ public RepositoryCommit addCommit(String repository, String branch, RevCommit commit) { RepositoryCommit commitModel = new RepositoryCommit(repository, branch, commit); + return addCommit(commitModel); + } + + /** + * Adds a commit to the activity object as long as the commit is not a + * duplicate. + * + * @param repository + * @param branch + * @param commit + * @return a RepositoryCommit, if one was added. Null if this is duplicate + * commit + */ + public RepositoryCommit addCommit(RepositoryCommit commitModel) { if (commits.add(commitModel)) { - String author = StringUtils.removeNewlines(commit.getAuthorIdent().getName()); + String author = StringUtils.removeNewlines(commitModel.getAuthorIdent().getName()); String authorName = author.toLowerCase(); - String authorEmail = StringUtils.removeNewlines(commit.getAuthorIdent().getEmailAddress()).toLowerCase(); - if (!repositoryMetrics.containsKey(repository)) { - repositoryMetrics.put(repository, new Metric(repository)); + String authorEmail = StringUtils.removeNewlines(commitModel.getAuthorIdent().getEmailAddress()).toLowerCase(); + if (!repositoryMetrics.containsKey(commitModel.repository)) { + repositoryMetrics.put(commitModel.repository, new Metric(commitModel.repository)); } - repositoryMetrics.get(repository).count++; + repositoryMetrics.get(commitModel.repository).count++; if (!authorExclusions.contains(authorName) && !authorExclusions.contains(authorEmail)) { if (!authorMetrics.containsKey(author)) { @@ -122,7 +136,7 @@ public class Activity implements Serializable, Comparable { } return null; } - + public int getCommitCount() { return commits.size(); } diff --git a/src/main/java/com/gitblit/models/RefLogEntry.java b/src/main/java/com/gitblit/models/RefLogEntry.java index 54d17771..79e8a2ca 100644 --- a/src/main/java/com/gitblit/models/RefLogEntry.java +++ b/src/main/java/com/gitblit/models/RefLogEntry.java @@ -160,7 +160,24 @@ public class RefLogEntry implements Serializable, Comparable { } return null; } - + + /** + * Adds a commit to the push entry object as long as the commit is not a + * duplicate. + * + * @param branch + * @param commit + * @return a RepositoryCommit, if one was added. Null if this is duplicate + * commit + */ + public RepositoryCommit addCommit(RepositoryCommit commit) { + if (commits.add(commit)) { + authorCount = -1; + return commit; + } + return null; + } + /** * Adds a a list of repository commits. This is used to construct discrete * ref push log entries diff --git a/src/main/java/com/gitblit/models/RepositoryCommit.java b/src/main/java/com/gitblit/models/RepositoryCommit.java index e68e8613..dd58b42b 100644 --- a/src/main/java/com/gitblit/models/RepositoryCommit.java +++ b/src/main/java/com/gitblit/models/RepositoryCommit.java @@ -17,8 +17,10 @@ package com.gitblit.models; import java.io.Serializable; import java.text.MessageFormat; +import java.util.Date; import java.util.List; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.revwalk.RevCommit; @@ -54,6 +56,10 @@ public class RepositoryCommit implements Serializable, Comparable> allRefs = JGitUtils - .getAllRefs(repository, model.showRemoteBranches); for (String branch : branches) { String shortName = branch; if (shortName.startsWith(Constants.R_HEADS)) { shortName = shortName.substring(Constants.R_HEADS.length()); } - List commits = JGitUtils.getRevLog(repository, - branch, thresholdDate); + List commits = CommitCache.instance().getCommits(model.name, repository, branch, thresholdDate); if (model.maxActivityCommits > 0 && commits.size() > model.maxActivityCommits) { // trim commits to maximum count commits = commits.subList(0, model.maxActivityCommits); } - for (RevCommit commit : commits) { - Date date = JGitUtils.getCommitDate(commit); + for (RepositoryCommit commit : commits) { + Date date = commit.getCommitDate(); String dateStr = df.format(date); if (!activity.containsKey(dateStr)) { // Normalize the date to midnight @@ -140,11 +135,7 @@ public class ActivityUtils { a.excludeAuthors(authorExclusions); activity.put(dateStr, a); } - RepositoryCommit commitModel = activity.get(dateStr) - .addCommit(model.name, shortName, commit); - if (commitModel != null) { - commitModel.setRefs(allRefs.get(commit.getId())); - } + activity.get(dateStr).addCommit(commit); } } diff --git a/src/main/java/com/gitblit/utils/CommitCache.java b/src/main/java/com/gitblit/utils/CommitCache.java new file mode 100644 index 00000000..e188ff9a --- /dev/null +++ b/src/main/java/com/gitblit/utils/CommitCache.java @@ -0,0 +1,253 @@ +/* + * Copyright 2013 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.utils; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.gitblit.models.RefModel; +import com.gitblit.models.RepositoryCommit; + +/** + * Caches repository commits for re-use in the dashboard and activity pages. + * + * @author James Moger + * + */ +public class CommitCache { + + private static final CommitCache instance; + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + protected final Map>> cache; + + protected int cacheDays = -1; + + public static CommitCache instance() { + return instance; + } + + static { + instance = new CommitCache(); + } + + protected CommitCache() { + cache = new ConcurrentHashMap>>(); + } + + /** + * Returns the cutoff date for the cache. Commits after this date are cached. + * Commits before this date are not cached. + * + * @return + */ + protected Date getCacheCutoffDate() { + final Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(System.currentTimeMillis()); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + cal.add(Calendar.DATE, -1*cacheDays); + return cal.getTime(); + } + + /** + * Sets the number of days to cache. + * + * @param days + */ + public synchronized void setCacheDays(int days) { + this.cacheDays = days; + clear(); + } + + /** + * Clears the entire commit cache. + * + */ + public void clear() { + cache.clear(); + } + + /** + * Clears the commit cache for a specific repository. + * + * @param repositoryName + */ + public void clear(String repositoryName) { + String repoKey = repositoryName.toLowerCase(); + ObjectCache> repoCache = cache.remove(repoKey); + if (repoCache != null) { + logger.info(MessageFormat.format("{0} commit cache cleared", repositoryName)); + } + } + + /** + * Get all commits for the specified repository:branch that are in the cache. + * + * @param repositoryName + * @param repository + * @param branch + * @return a list of commits + */ + public List getCommits(String repositoryName, Repository repository, String branch) { + return getCommits(repositoryName, repository, branch, getCacheCutoffDate()); + } + + /** + * Get all commits for the specified repository:branch since a specific date. + * These commits may be retrieved from the cache if the sinceDate is after + * the cacheCutoffDate. + * + * @param repositoryName + * @param repository + * @param branch + * @param sinceDate + * @return a list of commits + */ + public List getCommits(String repositoryName, Repository repository, String branch, Date sinceDate) { + long start = System.nanoTime(); + Date cacheCutoffDate = getCacheCutoffDate(); + List list; + if (cacheDays > 0 && (sinceDate.getTime() >= cacheCutoffDate.getTime())) { + // request fits within the cache window + String repoKey = repositoryName.toLowerCase(); + if (!cache.containsKey(repoKey)) { + cache.put(repoKey, new ObjectCache>()); + } + + ObjectCache> repoCache = cache.get(repoKey); + String branchKey = branch.toLowerCase(); + + RevCommit tip = JGitUtils.getCommit(repository, branch); + Date tipDate = JGitUtils.getCommitDate(tip); + + List commits; + if (!repoCache.hasCurrent(branchKey, tipDate)) { + commits = repoCache.getObject(branchKey); + if (ArrayUtils.isEmpty(commits)) { + // we don't have any cached commits for this branch, reload + commits = get(repositoryName, repository, branch, cacheCutoffDate); + repoCache.updateObject(branchKey, tipDate, commits); + logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs", + commits.size(), repositoryName, branch, cacheCutoffDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start))); + } else { + // incrementally update cache since the last cached commit + ObjectId sinceCommit = commits.get(0).getId(); + List incremental = get(repositoryName, repository, branch, sinceCommit); + logger.info(MessageFormat.format("incrementally added {0} commits to cache for {1}:{2} in {3} msecs", + incremental.size(), repositoryName, branch, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start))); + incremental.addAll(commits); + repoCache.updateObject(branchKey, tipDate, incremental); + commits = incremental; + } + } else { + // cache is current + commits = repoCache.getObject(branchKey); + // evict older commits outside the cache window + commits = reduce(commits, cacheCutoffDate); + // update cache + repoCache.updateObject(branchKey, tipDate, commits); + } + + if (sinceDate.equals(cacheCutoffDate)) { + list = commits; + } else { + // reduce the commits to those since the specified date + list = reduce(commits, sinceDate); + } + logger.debug(MessageFormat.format("retrieved {0} commits from cache of {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs", + list.size(), repositoryName, branch, sinceDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start))); + } else { + // not caching or request outside cache window + list = get(repositoryName, repository, branch, sinceDate); + logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs", + list.size(), repositoryName, branch, sinceDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start))); + } + return list; + } + + /** + * Returns a list of commits for the specified repository branch. + * + * @param repositoryName + * @param repository + * @param branch + * @param sinceDate + * @return a list of commits + */ + protected List get(String repositoryName, Repository repository, String branch, Date sinceDate) { + Map> allRefs = JGitUtils.getAllRefs(repository, false); + List commits = new ArrayList(); + for (RevCommit commit : JGitUtils.getRevLog(repository, branch, sinceDate)) { + RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit); + commitModel.setRefs(allRefs.get(commitModel.getName())); + commits.add(commitModel); + } + return commits; + } + + /** + * Returns a list of commits for the specified repository branch since the specified commit. + * + * @param repositoryName + * @param repository + * @param branch + * @param sinceCommit + * @return a list of commits + */ + protected List get(String repositoryName, Repository repository, String branch, ObjectId sinceCommit) { + Map> allRefs = JGitUtils.getAllRefs(repository, false); + List commits = new ArrayList(); + for (RevCommit commit : JGitUtils.getRevLog(repository, sinceCommit.getName(), branch)) { + RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit); + commitModel.setRefs(allRefs.get(commitModel.getName())); + commits.add(commitModel); + } + return commits; + } + + /** + * Reduces the list of commits to those since the specified date. + * + * @param commits + * @param sinceDate + * @return a list of commits + */ + protected List reduce(List commits, Date sinceDate) { + List filtered = new ArrayList(); + for (RepositoryCommit commit : commits) { + if (commit.getCommitDate().compareTo(sinceDate) >= 0) { + filtered.add(commit); + } + } + return filtered; + } +} diff --git a/src/main/java/com/gitblit/utils/ObjectCache.java b/src/main/java/com/gitblit/utils/ObjectCache.java index 38f2e59a..692669f4 100644 --- a/src/main/java/com/gitblit/utils/ObjectCache.java +++ b/src/main/java/com/gitblit/utils/ObjectCache.java @@ -85,7 +85,7 @@ public class ObjectCache implements Serializable { obj.object = object; } - public Object remove(String name) { + public X remove(String name) { if (cache.containsKey(name)) { return cache.remove(name).object; } diff --git a/src/main/java/com/gitblit/utils/RefLogUtils.java b/src/main/java/com/gitblit/utils/RefLogUtils.java index ce03a163..73b71468 100644 --- a/src/main/java/com/gitblit/utils/RefLogUtils.java +++ b/src/main/java/com/gitblit/utils/RefLogUtils.java @@ -549,15 +549,15 @@ public class RefLogUtils { String linearParent = null; for (RefModel local : JGitUtils.getLocalBranches(repository, true, -1)) { String branch = local.getName(); - List commits = JGitUtils.getRevLog(repository, branch, minimumDate); - for (RevCommit commit : commits) { + List commits = CommitCache.instance().getCommits(repositoryName, repository, branch, minimumDate); + for (RepositoryCommit commit : commits) { if (linearParent != null) { if (!commit.getName().equals(linearParent)) { // only follow linear branch commits continue; } } - Date date = JGitUtils.getCommitDate(commit); + Date date = commit.getCommitDate(); String dateStr = df.format(date); if (!dailydigests.containsKey(dateStr)) { dailydigests.put(dateStr, new DailyLogEntry(repositoryName, date)); @@ -571,7 +571,7 @@ public class RefLogUtils { digest.updateRef(branch, ReceiveCommand.Type.UPDATE, linearParent, commit.getName()); } - RepositoryCommit repoCommit = digest.addCommit(branch, commit); + RepositoryCommit repoCommit = digest.addCommit(commit); if (repoCommit != null) { List matchedRefs = allRefs.get(commit.getId()); repoCommit.setRefs(matchedRefs); @@ -587,7 +587,8 @@ public class RefLogUtils { } RefLogEntry tagEntry = tags.get(dateStr); tagEntry.updateRef(ref.getName(), ReceiveCommand.Type.CREATE); - tagEntry.addCommit(ref.getName(), commit); + RepositoryCommit rc = repoCommit.clone(ref.getName()); + tagEntry.addCommit(rc); } else if (ref.getName().startsWith(Constants.R_PULL)) { // treat pull requests as special events in the log if (!pulls.containsKey(dateStr)) { @@ -597,7 +598,8 @@ public class RefLogUtils { } RefLogEntry pullEntry = pulls.get(dateStr); pullEntry.updateRef(ref.getName(), ReceiveCommand.Type.CREATE); - pullEntry.addCommit(ref.getName(), commit); + RepositoryCommit rc = repoCommit.clone(ref.getName()); + pullEntry.addCommit(rc); } } } diff --git a/src/main/java/com/gitblit/wicket/panels/ActivityPanel.java b/src/main/java/com/gitblit/wicket/panels/ActivityPanel.java index 669c36be..b509f655 100644 --- a/src/main/java/com/gitblit/wicket/panels/ActivityPanel.java +++ b/src/main/java/com/gitblit/wicket/panels/ActivityPanel.java @@ -22,6 +22,7 @@ import org.apache.wicket.markup.html.link.BookmarkablePageLink; import org.apache.wicket.markup.repeater.Item; import org.apache.wicket.markup.repeater.data.DataView; import org.apache.wicket.markup.repeater.data.ListDataProvider; +import org.eclipse.jgit.lib.Repository; import com.gitblit.Constants; import com.gitblit.GitBlit; @@ -101,7 +102,7 @@ public class ActivityPanel extends BasePanel { commitItem.add(repositoryLink); // repository branch - LinkPanel branchLink = new LinkPanel("branch", "list", commit.branch, + LinkPanel branchLink = new LinkPanel("branch", "list", Repository.shortenRefName(commit.branch), LogPage.class, WicketUtils.newObjectParameter(commit.repository, commit.branch), true); WicketUtils.setCssStyle(branchLink, "color: #008000;"); -- cgit v1.2.3