- Setting to automatically create an user account based on an authenticated user principal from the servlet container (issue-246)\r
- Added WindowsUserService to authenticate users against Windows accounts (issue-250)\r
- Global and per-repository setting to exclude authors from metrics (issue-251)\r
+ - Added commit cache to improve Activity, Dashboard, and Project page generation times\r
- Added SalesForce.com user service\r
- Added simple star/unstar function to flag or bookmark interesting repositories\r
- Added Dashboard page which shows a news feed for starred repositories and offers a filterable list of repositories you care about\r
- { name: 'realm.salesforce.orgId', defaultValue: 0 }\r
- { name: 'realm.windows.defaultDomain', defaultValue: ' ' }\r
- { name: 'realm.windows.backingUserService', defaultValue: 'users.conf' }\r
- - { name: 'web.activityDurationChoices', defaultValue: '7 14 28 60 90 180' }\r
+ - { name: 'web.activityDurationChoices', defaultValue: '7 14 21 28' }\r
+ - { name: 'web.activityCacheDays', defaultValue: 14 }\r
- { name: 'web.allowAppCloneLinks', defaultValue: 'true' }\r
- { name: 'web.forceDefaultLocale', defaultValue: ' ' }\r
- { name: 'web.metricAuthorExclusions', defaultValue: ' ' }\r
#\r
# SPACE-DELIMITED\r
# SINCE 1.3.0\r
-web.activityDurationChoices = 7 14 28 60 90 180\r
+web.activityDurationChoices = 7 14 21 28\r
+\r
+# The number of days of commits to cache in memory for the dashboard, activity,\r
+# and project pages. A value of 0 will disable all caching and will parse commits\r
+# in each repository per-request. If the value > 0 these pages will try to fulfill\r
+# requests using the commit cache. If the request specifies a period which falls\r
+# outside the commit cache window, then the cache will be ignored and the request\r
+# will be fulfilled by brute-force parsing all relevant commits per-repository.\r
+#\r
+# Consider the values specified for *web.activityDurationChoices* when setting\r
+# the cache size AND consider adjusting the JVM -Xmx heap parameter appropriately.\r
+#\r
+# SINCE 1.3.0\r
+# RESTART REQUIRED\r
+web.activityCacheDays = 14\r
\r
# Case-insensitive list of authors to exclude from metrics. Useful for\r
# eliminating bots.\r
import com.gitblit.utils.ArrayUtils;
import com.gitblit.utils.Base64;
import com.gitblit.utils.ByteFormat;
+import com.gitblit.utils.CommitCache;
import com.gitblit.utils.ContainerUtils;
import com.gitblit.utils.DeepCopier;
import com.gitblit.utils.FederationUtils;
configureFanout();
configureGitDaemon();
+ CommitCache.instance().setCacheDays(settings.getInteger(Keys.web.activityCacheDays, 14));
+
ContainerUtils.CVE_2007_0450.test();
}
*/\r
public RepositoryCommit addCommit(String repository, String branch, RevCommit commit) {\r
RepositoryCommit commitModel = new RepositoryCommit(repository, branch, commit);\r
+ return addCommit(commitModel);\r
+ }\r
+\r
+ /**\r
+ * Adds a commit to the activity object as long as the commit is not a\r
+ * duplicate.\r
+ * \r
+ * @param repository\r
+ * @param branch\r
+ * @param commit\r
+ * @return a RepositoryCommit, if one was added. Null if this is duplicate\r
+ * commit\r
+ */\r
+ public RepositoryCommit addCommit(RepositoryCommit commitModel) {\r
if (commits.add(commitModel)) {\r
- String author = StringUtils.removeNewlines(commit.getAuthorIdent().getName());\r
+ String author = StringUtils.removeNewlines(commitModel.getAuthorIdent().getName());\r
String authorName = author.toLowerCase();\r
- String authorEmail = StringUtils.removeNewlines(commit.getAuthorIdent().getEmailAddress()).toLowerCase();\r
- if (!repositoryMetrics.containsKey(repository)) {\r
- repositoryMetrics.put(repository, new Metric(repository));\r
+ String authorEmail = StringUtils.removeNewlines(commitModel.getAuthorIdent().getEmailAddress()).toLowerCase();\r
+ if (!repositoryMetrics.containsKey(commitModel.repository)) {\r
+ repositoryMetrics.put(commitModel.repository, new Metric(commitModel.repository));\r
}\r
- repositoryMetrics.get(repository).count++;\r
+ repositoryMetrics.get(commitModel.repository).count++;\r
\r
if (!authorExclusions.contains(authorName) && !authorExclusions.contains(authorEmail)) {\r
if (!authorMetrics.containsKey(author)) {\r
}\r
return null;\r
}\r
- \r
+\r
public int getCommitCount() {\r
return commits.size();\r
}\r
}\r
return null;\r
}\r
- \r
+\r
+ /**\r
+ * Adds a commit to the push entry object as long as the commit is not a\r
+ * duplicate.\r
+ * \r
+ * @param branch\r
+ * @param commit\r
+ * @return a RepositoryCommit, if one was added. Null if this is duplicate\r
+ * commit\r
+ */\r
+ public RepositoryCommit addCommit(RepositoryCommit commit) {\r
+ if (commits.add(commit)) {\r
+ authorCount = -1;\r
+ return commit;\r
+ }\r
+ return null;\r
+ }\r
+\r
/**\r
* Adds a a list of repository commits. This is used to construct discrete\r
* ref push log entries\r
import java.io.Serializable;
import java.text.MessageFormat;
+import java.util.Date;
import java.util.List;
+import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.revwalk.RevCommit;
return refs;
}
+ public ObjectId getId() {
+ return commit.getId();
+ }
+
public String getName() {
return commit.getName();
}
public String getShortMessage() {
return commit.getShortMessage();
}
+
+ public Date getCommitDate() {
+ return new Date(commit.getCommitTime() * 1000L);
+ }
public int getParentCount() {
return commit.getParentCount();
}
+
+ public RevCommit [] getParents() {
+ return commit.getParents();
+ }
public PersonIdent getAuthorIdent() {
return commit.getAuthorIdent();
public PersonIdent getCommitterIdent() {
return commit.getCommitterIdent();
}
-
+
@Override
public boolean equals(Object o) {
if (o instanceof RepositoryCommit) {
return 0;
}
+ public RepositoryCommit clone(String withRef) {
+ return new RepositoryCommit(repository, withRef, commit);
+ }
+
@Override
public String toString() {
return MessageFormat.format("{0} {1} {2,date,yyyy-MM-dd HH:mm} {3} {4}",
import java.util.TreeSet;\r
\r
import org.eclipse.jgit.lib.Constants;\r
-import org.eclipse.jgit.lib.ObjectId;\r
import org.eclipse.jgit.lib.Repository;\r
-import org.eclipse.jgit.revwalk.RevCommit;\r
\r
import com.gitblit.GitBlit;\r
import com.gitblit.Keys;\r
} else {\r
branches.add(objectId);\r
}\r
- Map<ObjectId, List<RefModel>> allRefs = JGitUtils\r
- .getAllRefs(repository, model.showRemoteBranches);\r
\r
for (String branch : branches) {\r
String shortName = branch;\r
if (shortName.startsWith(Constants.R_HEADS)) {\r
shortName = shortName.substring(Constants.R_HEADS.length());\r
}\r
- List<RevCommit> commits = JGitUtils.getRevLog(repository,\r
- branch, thresholdDate);\r
+ List<RepositoryCommit> commits = CommitCache.instance().getCommits(model.name, repository, branch, thresholdDate);\r
if (model.maxActivityCommits > 0 && commits.size() > model.maxActivityCommits) {\r
// trim commits to maximum count\r
commits = commits.subList(0, model.maxActivityCommits);\r
}\r
- for (RevCommit commit : commits) { \r
- Date date = JGitUtils.getCommitDate(commit);\r
+ for (RepositoryCommit commit : commits) { \r
+ Date date = commit.getCommitDate();\r
String dateStr = df.format(date);\r
if (!activity.containsKey(dateStr)) {\r
// Normalize the date to midnight\r
a.excludeAuthors(authorExclusions);\r
activity.put(dateStr, a);\r
}\r
- RepositoryCommit commitModel = activity.get(dateStr)\r
- .addCommit(model.name, shortName, commit);\r
- if (commitModel != null) {\r
- commitModel.setRefs(allRefs.get(commit.getId()));\r
- }\r
+ activity.get(dateStr).addCommit(commit);\r
}\r
}\r
\r
--- /dev/null
+/*
+ * Copyright 2013 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.utils;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.gitblit.models.RefModel;
+import com.gitblit.models.RepositoryCommit;
+
+/**
+ * Caches repository commits for re-use in the dashboard and activity pages.
+ *
+ * @author James Moger
+ *
+ */
+public class CommitCache {
+
+ private static final CommitCache instance;
+
+ protected final Logger logger = LoggerFactory.getLogger(getClass());
+
+ protected final Map<String, ObjectCache<List<RepositoryCommit>>> cache;
+
+ protected int cacheDays = -1;
+
+ public static CommitCache instance() {
+ return instance;
+ }
+
+ static {
+ instance = new CommitCache();
+ }
+
+ protected CommitCache() {
+ cache = new ConcurrentHashMap<String, ObjectCache<List<RepositoryCommit>>>();
+ }
+
+ /**
+ * Returns the cutoff date for the cache. Commits after this date are cached.
+ * Commits before this date are not cached.
+ *
+ * @return
+ */
+ protected Date getCacheCutoffDate() {
+ final Calendar cal = Calendar.getInstance();
+ cal.setTimeInMillis(System.currentTimeMillis());
+ cal.set(Calendar.HOUR_OF_DAY, 0);
+ cal.set(Calendar.MINUTE, 0);
+ cal.set(Calendar.SECOND, 0);
+ cal.set(Calendar.MILLISECOND, 0);
+ cal.add(Calendar.DATE, -1*cacheDays);
+ return cal.getTime();
+ }
+
+ /**
+ * Sets the number of days to cache.
+ *
+ * @param days
+ */
+ public synchronized void setCacheDays(int days) {
+ this.cacheDays = days;
+ clear();
+ }
+
+ /**
+ * Clears the entire commit cache.
+ *
+ */
+ public void clear() {
+ cache.clear();
+ }
+
+ /**
+ * Clears the commit cache for a specific repository.
+ *
+ * @param repositoryName
+ */
+ public void clear(String repositoryName) {
+ String repoKey = repositoryName.toLowerCase();
+ ObjectCache<List<RepositoryCommit>> repoCache = cache.remove(repoKey);
+ if (repoCache != null) {
+ logger.info(MessageFormat.format("{0} commit cache cleared", repositoryName));
+ }
+ }
+
+ /**
+ * Get all commits for the specified repository:branch that are in the cache.
+ *
+ * @param repositoryName
+ * @param repository
+ * @param branch
+ * @return a list of commits
+ */
+ public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch) {
+ return getCommits(repositoryName, repository, branch, getCacheCutoffDate());
+ }
+
+ /**
+ * Get all commits for the specified repository:branch since a specific date.
+ * These commits may be retrieved from the cache if the sinceDate is after
+ * the cacheCutoffDate.
+ *
+ * @param repositoryName
+ * @param repository
+ * @param branch
+ * @param sinceDate
+ * @return a list of commits
+ */
+ public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch, Date sinceDate) {
+ long start = System.nanoTime();
+ Date cacheCutoffDate = getCacheCutoffDate();
+ List<RepositoryCommit> list;
+ if (cacheDays > 0 && (sinceDate.getTime() >= cacheCutoffDate.getTime())) {
+ // request fits within the cache window
+ String repoKey = repositoryName.toLowerCase();
+ if (!cache.containsKey(repoKey)) {
+ cache.put(repoKey, new ObjectCache<List<RepositoryCommit>>());
+ }
+
+ ObjectCache<List<RepositoryCommit>> repoCache = cache.get(repoKey);
+ String branchKey = branch.toLowerCase();
+
+ RevCommit tip = JGitUtils.getCommit(repository, branch);
+ Date tipDate = JGitUtils.getCommitDate(tip);
+
+ List<RepositoryCommit> commits;
+ if (!repoCache.hasCurrent(branchKey, tipDate)) {
+ commits = repoCache.getObject(branchKey);
+ if (ArrayUtils.isEmpty(commits)) {
+ // we don't have any cached commits for this branch, reload
+ commits = get(repositoryName, repository, branch, cacheCutoffDate);
+ repoCache.updateObject(branchKey, tipDate, commits);
+ logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
+ commits.size(), repositoryName, branch, cacheCutoffDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
+ } else {
+ // incrementally update cache since the last cached commit
+ ObjectId sinceCommit = commits.get(0).getId();
+ List<RepositoryCommit> incremental = get(repositoryName, repository, branch, sinceCommit);
+ logger.info(MessageFormat.format("incrementally added {0} commits to cache for {1}:{2} in {3} msecs",
+ incremental.size(), repositoryName, branch, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
+ incremental.addAll(commits);
+ repoCache.updateObject(branchKey, tipDate, incremental);
+ commits = incremental;
+ }
+ } else {
+ // cache is current
+ commits = repoCache.getObject(branchKey);
+ // evict older commits outside the cache window
+ commits = reduce(commits, cacheCutoffDate);
+ // update cache
+ repoCache.updateObject(branchKey, tipDate, commits);
+ }
+
+ if (sinceDate.equals(cacheCutoffDate)) {
+ list = commits;
+ } else {
+ // reduce the commits to those since the specified date
+ list = reduce(commits, sinceDate);
+ }
+ logger.debug(MessageFormat.format("retrieved {0} commits from cache of {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
+ list.size(), repositoryName, branch, sinceDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
+ } else {
+ // not caching or request outside cache window
+ list = get(repositoryName, repository, branch, sinceDate);
+ logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
+ list.size(), repositoryName, branch, sinceDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
+ }
+ return list;
+ }
+
+ /**
+ * Returns a list of commits for the specified repository branch.
+ *
+ * @param repositoryName
+ * @param repository
+ * @param branch
+ * @param sinceDate
+ * @return a list of commits
+ */
+ protected List<RepositoryCommit> get(String repositoryName, Repository repository, String branch, Date sinceDate) {
+ Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository, false);
+ List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>();
+ for (RevCommit commit : JGitUtils.getRevLog(repository, branch, sinceDate)) {
+ RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit);
+ commitModel.setRefs(allRefs.get(commitModel.getName()));
+ commits.add(commitModel);
+ }
+ return commits;
+ }
+
+ /**
+ * Returns a list of commits for the specified repository branch since the specified commit.
+ *
+ * @param repositoryName
+ * @param repository
+ * @param branch
+ * @param sinceCommit
+ * @return a list of commits
+ */
+ protected List<RepositoryCommit> get(String repositoryName, Repository repository, String branch, ObjectId sinceCommit) {
+ Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository, false);
+ List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>();
+ for (RevCommit commit : JGitUtils.getRevLog(repository, sinceCommit.getName(), branch)) {
+ RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit);
+ commitModel.setRefs(allRefs.get(commitModel.getName()));
+ commits.add(commitModel);
+ }
+ return commits;
+ }
+
+ /**
+ * Reduces the list of commits to those since the specified date.
+ *
+ * @param commits
+ * @param sinceDate
+ * @return a list of commits
+ */
+ protected List<RepositoryCommit> reduce(List<RepositoryCommit> commits, Date sinceDate) {
+ List<RepositoryCommit> filtered = new ArrayList<RepositoryCommit>();
+ for (RepositoryCommit commit : commits) {
+ if (commit.getCommitDate().compareTo(sinceDate) >= 0) {
+ filtered.add(commit);
+ }
+ }
+ return filtered;
+ }
+}
obj.object = object;\r
}\r
\r
- public Object remove(String name) {\r
+ public X remove(String name) {\r
if (cache.containsKey(name)) {\r
return cache.remove(name).object;\r
}\r
String linearParent = null;
for (RefModel local : JGitUtils.getLocalBranches(repository, true, -1)) {
String branch = local.getName();
- List<RevCommit> commits = JGitUtils.getRevLog(repository, branch, minimumDate);
- for (RevCommit commit : commits) {
+ List<RepositoryCommit> commits = CommitCache.instance().getCommits(repositoryName, repository, branch, minimumDate);
+ for (RepositoryCommit commit : commits) {
if (linearParent != null) {
if (!commit.getName().equals(linearParent)) {
// only follow linear branch commits
continue;
}
}
- Date date = JGitUtils.getCommitDate(commit);
+ Date date = commit.getCommitDate();
String dateStr = df.format(date);
if (!dailydigests.containsKey(dateStr)) {
dailydigests.put(dateStr, new DailyLogEntry(repositoryName, date));
digest.updateRef(branch, ReceiveCommand.Type.UPDATE, linearParent, commit.getName());
}
- RepositoryCommit repoCommit = digest.addCommit(branch, commit);
+ RepositoryCommit repoCommit = digest.addCommit(commit);
if (repoCommit != null) {
List<RefModel> matchedRefs = allRefs.get(commit.getId());
repoCommit.setRefs(matchedRefs);
}
RefLogEntry tagEntry = tags.get(dateStr);
tagEntry.updateRef(ref.getName(), ReceiveCommand.Type.CREATE);
- tagEntry.addCommit(ref.getName(), commit);
+ RepositoryCommit rc = repoCommit.clone(ref.getName());
+ tagEntry.addCommit(rc);
} else if (ref.getName().startsWith(Constants.R_PULL)) {
// treat pull requests as special events in the log
if (!pulls.containsKey(dateStr)) {
}
RefLogEntry pullEntry = pulls.get(dateStr);
pullEntry.updateRef(ref.getName(), ReceiveCommand.Type.CREATE);
- pullEntry.addCommit(ref.getName(), commit);
+ RepositoryCommit rc = repoCommit.clone(ref.getName());
+ pullEntry.addCommit(rc);
}
}
}
import org.apache.wicket.markup.repeater.Item;\r
import org.apache.wicket.markup.repeater.data.DataView;\r
import org.apache.wicket.markup.repeater.data.ListDataProvider;\r
+import org.eclipse.jgit.lib.Repository;\r
\r
import com.gitblit.Constants;\r
import com.gitblit.GitBlit;\r
commitItem.add(repositoryLink);\r
\r
// repository branch\r
- LinkPanel branchLink = new LinkPanel("branch", "list", commit.branch,\r
+ LinkPanel branchLink = new LinkPanel("branch", "list", Repository.shortenRefName(commit.branch),\r
LogPage.class, WicketUtils.newObjectParameter(commit.repository,\r
commit.branch), true);\r
WicketUtils.setCssStyle(branchLink, "color: #008000;");\r
projects / gitblit.git / commitdiff