# RESTART REQUIRED\r
git.repositoriesFolder = git\r
\r
+# Build the available repository list at startup and cache this list for reuse.\r
+# This reduces disk io when presenting the repositories page, responding to rpcs,\r
+# etc, but it means that Gitblit will not automatically identify repositories\r
+# added or deleted by external tools.\r
+#\r
+# For this case you can use curl, wget, etc to issue an rpc request to clear the\r
+# cache (e.g. https://localhost/rpc?req=CLEAR_REPOSITORY_CACHE)\r
+#\r
+# SINCE 1.1.0\r
+git.cacheRepositoryList = true\r
+\r
# Search the repositories folder subfolders for other repositories.\r
# Repositories MAY NOT be nested (i.e. one repository within another)\r
# but they may be grouped together in subfolders.\r
# Maximum number of folders to recurse into when searching for repositories.\r
# The default value, -1, disables depth limits.\r
#\r
-# SINCE 1.0.1\r
+# SINCE 1.1.0\r
git.searchRecursionDepth = -1\r
\r
# List of regex exclusion patterns to match against folders found in\r
#\r
# SPACE-DELIMITED\r
# CASE-SENSITIVE\r
-# SINCE 1.0.1\r
+# SINCE 1.1.0\r
git.searchExclusions =\r
\r
# List of regex url patterns for extracting a repository name when locating\r
#\r
# SPACE-DELIMITED\r
# CASE-SENSITIVE\r
-# SINCE 1.0.1\r
+# SINCE 1.1.0\r
git.submoduleUrlPatterns = .*?://github.com/(.*)\r
\r
# Allow push/pull over http/https with JGit servlet.\r
# AUTHENTICATED = any authenticated user is granted restricted access\r
# NAMED = only named users/teams are granted restricted access\r
#\r
-# SINCE 1.0.1\r
+# SINCE 1.1.0\r
git.defaultAuthorizationControl = NAMED\r
\r
# Number of bytes of a pack file to load into memory in a single read operation.\r
<tr><th>Release</th><th>Protocol Version</th></tr>\r
<tr><td>Gitblit v0.7.0</td><td>1 (inferred version)</td></tr>\r
<tr><td>Gitblit v0.8.0</td><td>2</td></tr>\r
-<tr><td>Gitblit v0.9.0+</td><td>3</td></tr>\r
+<tr><td>Gitblit v0.9.0 - v1.0.0</td><td>3</td></tr>\r
+<tr><td>Gitblit v1.1.0+</td><td>4</td></tr>\r
</tbody>\r
</table>\r
\r
<tr><td>LIST_SETTINGS</td><td>-</td><td><em>admin</em></td><td>1</td><td>-</td><td>ServerSettings (all keys)</td></tr>\r
<tr><td>EDIT_SETTINGS</td><td>-</td><td><em>admin</em></td><td>1</td><td>Map<String, String></td><td>-</td></tr>\r
<tr><td>LIST_STATUS</td><td>-</td><td><em>admin</em></td><td>1</td><td>-</td><td>ServerStatus (see example below)</td></tr>\r
+<tr><td>CLEAR_REPOSITORY_CACHE</td><td>-</td><td><em>admin</em></td><td>4</td><td>-</td><td>-</td></tr>\r
</table>\r
\r
### RPC/HTTP Response Codes\r
\r
#### additions\r
\r
+- Identified repository list is now cached by default to reduce disk io and to improve performance (issue 103) \r
+ **New:** *git.cacheRepositoryList=true*\r
- Preliminary bare repository submodule support \r
**New:** *git.submoduleUrlPatterns=*\r
- *git.submoduleUrlPatterns* is a space-delimited list of regular expressions for extracting a repository name from a submodule url. \r
\r
// The build script extracts this exact line so be careful editing it\r
// and only use A-Z a-z 0-9 .-_ in the string.\r
- public static final String VERSION = "1.0.1-SNAPSHOT";\r
+ public static final String VERSION = "1.1.0-SNAPSHOT";\r
\r
// The build script extracts this exact line so be careful editing it\r
// and only use A-Z a-z 0-9 .-_ in the string.\r
LIST_TEAMS, CREATE_TEAM, EDIT_TEAM, DELETE_TEAM,\r
LIST_REPOSITORY_MEMBERS, SET_REPOSITORY_MEMBERS, LIST_REPOSITORY_TEAMS, SET_REPOSITORY_TEAMS, \r
LIST_FEDERATION_REGISTRATIONS, LIST_FEDERATION_RESULTS, LIST_FEDERATION_PROPOSALS, LIST_FEDERATION_SETS,\r
- EDIT_SETTINGS, LIST_STATUS;\r
+ EDIT_SETTINGS, LIST_STATUS, CLEAR_REPOSITORY_CACHE;\r
\r
public static RpcRequest fromName(String name) {\r
for (RpcRequest type : values()) {\r
import java.util.TimeZone;\r
import java.util.TreeSet;\r
import java.util.concurrent.ConcurrentHashMap;\r
+import java.util.concurrent.CopyOnWriteArrayList;\r
import java.util.concurrent.Executors;\r
import java.util.concurrent.ScheduledExecutorService;\r
import java.util.concurrent.TimeUnit;\r
import java.util.concurrent.atomic.AtomicInteger;\r
+import java.util.concurrent.atomic.AtomicReference;\r
\r
import javax.mail.Message;\r
import javax.mail.MessagingException;\r
private final ObjectCache<Long> repositorySizeCache = new ObjectCache<Long>();\r
\r
private final ObjectCache<List<Metric>> repositoryMetricsCache = new ObjectCache<List<Metric>>();\r
+ \r
+ private final List<String> repositoryListCache = new CopyOnWriteArrayList<String>();\r
+ \r
+ private final AtomicReference<String> repositoryListSettingsChecksum = new AtomicReference<String>("");\r
\r
private RepositoryResolver<Void> repositoryResolver;\r
\r
public boolean deleteTeam(String teamname) {\r
return userService.deleteTeam(teamname);\r
}\r
+ \r
+ /**\r
+ * Adds the repository to the list of cached repositories if Gitblit is\r
+ * configured to cache the repository list.\r
+ * \r
+ * @param name\r
+ */\r
+ private void addToCachedRepositoryList(String name) {\r
+ if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) {\r
+ repositoryListCache.add(name);\r
+ }\r
+ }\r
\r
/**\r
* Clears all the cached data for the specified repository.\r
* \r
* @param repositoryName\r
+ * @param isDeleted\r
*/\r
- public void clearRepositoryCache(String repositoryName) {\r
+ private void clearRepositoryCache(String repositoryName, boolean isDeleted) {\r
repositorySizeCache.remove(repositoryName);\r
repositoryMetricsCache.remove(repositoryName);\r
+ \r
+ if (isDeleted) {\r
+ repositoryListCache.remove(repositoryName);\r
+ }\r
+ }\r
+ \r
+ /**\r
+ * Resets the repository list cache.\r
+ * \r
+ */\r
+ public void resetRepositoryListCache() {\r
+ logger.info("Repository cache manually reset");\r
+ repositoryListCache.clear();\r
+ }\r
+ \r
+ /**\r
+ * Calculate the checksum of settings that affect the repository list cache.\r
+ * @return a checksum\r
+ */\r
+ private String getRepositoryListSettingsChecksum() {\r
+ StringBuilder ns = new StringBuilder();\r
+ ns.append(settings.getString(Keys.git.cacheRepositoryList, "")).append('\n');\r
+ ns.append(settings.getString(Keys.git.onlyAccessBareRepositories, "")).append('\n');\r
+ ns.append(settings.getString(Keys.git.searchRepositoriesSubfolders, "")).append('\n');\r
+ ns.append(settings.getString(Keys.git.searchRecursionDepth, "")).append('\n');\r
+ ns.append(settings.getString(Keys.git.searchExclusions, "")).append('\n');\r
+ String checksum = StringUtils.getSHA1(ns.toString());\r
+ return checksum;\r
+ }\r
+ \r
+ /**\r
+ * Compare the last repository list setting checksum to the current checksum.\r
+ * If different then clear the cache so that it may be rebuilt.\r
+ * \r
+ * @return true if the cached repository list is valid since the last check\r
+ */\r
+ private boolean isValidRepositoryList() {\r
+ String newChecksum = getRepositoryListSettingsChecksum();\r
+ boolean valid = newChecksum.equals(repositoryListSettingsChecksum.get());\r
+ repositoryListSettingsChecksum.set(newChecksum);\r
+ if (!valid && settings.getBoolean(Keys.git.cacheRepositoryList, true)) {\r
+ logger.info("Repository list settings have changed. Clearing repository list cache.");\r
+ repositoryListCache.clear();\r
+ }\r
+ return valid;\r
}\r
\r
/**\r
* @return list of all repositories\r
*/\r
public List<String> getRepositoryList() {\r
- return JGitUtils.getRepositoryList(repositoriesFolder, \r
- settings.getBoolean(Keys.git.onlyAccessBareRepositories, false),\r
- settings.getBoolean(Keys.git.searchRepositoriesSubfolders, true),\r
- settings.getInteger(Keys.git.searchRecursionDepth, -1),\r
- settings.getStrings(Keys.git.searchExclusions));\r
+ if (repositoryListCache.size() == 0 || !isValidRepositoryList()) {\r
+ // we are not caching OR we have not yet cached OR the cached list is invalid\r
+ long startTime = System.currentTimeMillis();\r
+ List<String> repositories = JGitUtils.getRepositoryList(repositoriesFolder, \r
+ settings.getBoolean(Keys.git.onlyAccessBareRepositories, false),\r
+ settings.getBoolean(Keys.git.searchRepositoriesSubfolders, true),\r
+ settings.getInteger(Keys.git.searchRecursionDepth, -1),\r
+ settings.getStrings(Keys.git.searchExclusions));\r
+\r
+ if (!settings.getBoolean(Keys.git.cacheRepositoryList, true)) {\r
+ // we are not caching\r
+ StringUtils.sortRepositorynames(repositories);\r
+ return repositories;\r
+ } else {\r
+ // we are caching this list\r
+ String msg = "{0} repositories identified in {1} msecs";\r
+\r
+ // optionally (re)calculate repository sizes\r
+ if (getBoolean(Keys.web.showRepositorySizes, true)) {\r
+ msg = "{0} repositories identified with calculated folder sizes in {1} msecs";\r
+ for (String repository : repositories) {\r
+ RepositoryModel model = getRepositoryModel(repository);\r
+ if (!model.skipSizeCalculation) {\r
+ calculateSize(model);\r
+ }\r
+ }\r
+ }\r
+ \r
+ // update cache\r
+ repositoryListCache.addAll(repositories);\r
+ \r
+ long duration = System.currentTimeMillis() - startTime;\r
+ logger.info(MessageFormat.format(msg, repositoryListCache.size(), duration));\r
+ }\r
+ }\r
+ \r
+ // return sorted copy of cached list\r
+ List<String> list = new ArrayList<String>(repositoryListCache); \r
+ StringUtils.sortRepositorynames(list);\r
+ return list;\r
}\r
\r
/**\r
* @return list of repository models accessible to user\r
*/\r
public List<RepositoryModel> getRepositoryModels(UserModel user) {\r
+ long methodStart = System.currentTimeMillis();\r
List<String> list = getRepositoryList();\r
List<RepositoryModel> repositories = new ArrayList<RepositoryModel>();\r
for (String repo : list) {\r
}\r
}\r
long duration = System.currentTimeMillis() - startTime;\r
- logger.info(MessageFormat.format("{0} repository sizes calculated in {1} msecs",\r
+ if (duration > 250) {\r
+ // only log calcualtion time if > 250 msecs\r
+ logger.info(MessageFormat.format("{0} repository sizes calculated in {1} msecs",\r
repoCount, duration));\r
+ }\r
}\r
+ long duration = System.currentTimeMillis() - methodStart;\r
+ logger.info(MessageFormat.format("{0} repository models loaded for {1} in {2} msecs",\r
+ repositories.size(), user.username, duration));\r
return repositories;\r
}\r
\r
model.hasCommits = JGitUtils.hasCommits(r);\r
model.lastChange = JGitUtils.getLastChange(r);\r
model.isBare = r.isBare();\r
- StoredConfig config = JGitUtils.readConfig(r);\r
+ \r
+ StoredConfig config = r.getConfig();\r
if (config != null) {\r
model.description = getConfig(config, "description", "");\r
model.owner = getConfig(config, "owner", "");\r
// create repository\r
logger.info("create repository " + repository.name);\r
r = JGitUtils.createRepository(repositoriesFolder, repository.name);\r
+ \r
+ // add name to cache\r
+ addToCachedRepositoryList(repository.name);\r
} else {\r
// rename repository\r
if (!repositoryName.equalsIgnoreCase(repository.name)) {\r
}\r
\r
// clear the cache\r
- clearRepositoryCache(repositoryName);\r
+ clearRepositoryCache(repositoryName, true);\r
+ \r
+ // add new name to repository list cache\r
+ addToCachedRepositoryList(repository.name);\r
}\r
\r
// load repository\r
repository.name, currentRef, repository.HEAD));\r
if (JGitUtils.setHEADtoRef(r, repository.HEAD)) {\r
// clear the cache\r
- clearRepositoryCache(repository.name);\r
+ clearRepositoryCache(repository.name, false);\r
}\r
}\r
\r
r.close();\r
}\r
}\r
-\r
+ \r
/**\r
* Updates the Gitblit configuration for the specified repository.\r
* \r
* the Gitblit repository model\r
*/\r
public void updateConfiguration(Repository r, RepositoryModel repository) {\r
- StoredConfig config = JGitUtils.readConfig(r);\r
+ StoredConfig config = r.getConfig();\r
config.setString(Constants.CONFIG_GITBLIT, null, "description", repository.description);\r
config.setString(Constants.CONFIG_GITBLIT, null, "owner", repository.owner);\r
config.setBoolean(Constants.CONFIG_GITBLIT, null, "useTickets", repository.useTickets);\r
public boolean deleteRepository(String repositoryName) {\r
try {\r
closeRepository(repositoryName);\r
+ // clear the repository cache\r
+ clearRepositoryCache(repositoryName, true); \r
+\r
File folder = new File(repositoriesFolder, repositoryName);\r
if (folder.exists() && folder.isDirectory()) {\r
FileUtils.delete(folder, FileUtils.RECURSIVE | FileUtils.RETRY);\r
return true;\r
}\r
}\r
-\r
- // clear the repository cache\r
- clearRepositoryCache(repositoryName);\r
} catch (Throwable t) {\r
logger.error(MessageFormat.format("Failed to delete repository {0}", repositoryName), t);\r
}\r
repositoriesFolder = getRepositoriesFolder();\r
logger.info("Git repositories folder " + repositoriesFolder.getAbsolutePath());\r
repositoryResolver = new FileResolver<Void>(repositoriesFolder, true);\r
+\r
+ // calculate repository list settings checksum for future config changes\r
+ repositoryListSettingsChecksum.set(getRepositoryListSettingsChecksum());\r
+\r
+ // build initial repository list\r
+ if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) {\r
+ logger.info("Identifying available repositories...");\r
+ getRepositoryList();\r
+ }\r
\r
logTimezone("JVM", TimeZone.getDefault());\r
logTimezone(Constants.NAME, getTimezone());\r
\r
private static final long serialVersionUID = 1L;\r
\r
- public static final int PROTOCOL_VERSION = 3;\r
+ public static final int PROTOCOL_VERSION = 4;\r
\r
public RpcServlet() {\r
super();\r
} else {\r
response.sendError(notAllowedCode);\r
}\r
+ } else if (RpcRequest.CLEAR_REPOSITORY_CACHE.equals(reqType)) {\r
+ // clear the repository list cache\r
+ if (allowAdmin) {\r
+ GitBlit.self().resetRepositoryListCache();\r
+ } else {\r
+ response.sendError(notAllowedCode);\r
+ }\r
}\r
\r
// send the result of the request\r
public boolean deleteRepository(RepositoryModel repository) throws IOException {\r
return RpcUtils.deleteRepository(repository, url, account, password);\r
}\r
+ \r
+ public boolean clearRepositoryCache() throws IOException {\r
+ return RpcUtils.clearRepositoryCache(url, account, password);\r
+ }\r
\r
public boolean createUser(UserModel user) throws IOException {\r
return RpcUtils.createUser(user, url, account, password);\r
\r
private JTextField filterTextfield;\r
\r
+ private JButton clearCache;\r
+\r
public RepositoriesPanel(GitblitClient gitblit) {\r
super();\r
this.gitblit = gitblit;\r
refreshRepositories();\r
}\r
});\r
+ \r
+ clearCache = new JButton(Translation.get("gb.clearCache"));\r
+ clearCache.addActionListener(new ActionListener() {\r
+ public void actionPerformed(ActionEvent e) {\r
+ clearCache();\r
+ }\r
+ });\r
\r
createRepository = new JButton(Translation.get("gb.create"));\r
createRepository.addActionListener(new ActionListener() {\r
repositoryTablePanel.add(new JScrollPane(table), BorderLayout.CENTER);\r
\r
JPanel repositoryControls = new JPanel(new FlowLayout(FlowLayout.CENTER, Utils.MARGIN, 0));\r
+ repositoryControls.add(clearCache);\r
repositoryControls.add(refreshRepositories);\r
repositoryControls.add(browseRepository);\r
repositoryControls.add(createRepository);\r
protected abstract void updateTeamsTable();\r
\r
protected void disableManagement() {\r
+ clearCache.setVisible(false);\r
createRepository.setVisible(false);\r
editRepository.setVisible(false);\r
delRepository.setVisible(false);\r
};\r
worker.execute();\r
}\r
+ \r
+ protected void clearCache() {\r
+ GitblitWorker worker = new GitblitWorker(RepositoriesPanel.this,\r
+ RpcRequest.CLEAR_REPOSITORY_CACHE) {\r
+ @Override\r
+ protected Boolean doRequest() throws IOException {\r
+ if (gitblit.clearRepositoryCache()) {\r
+ gitblit.refreshRepositories();\r
+ return true;\r
+ }\r
+ return false;\r
+ }\r
+\r
+ @Override\r
+ protected void onSuccess() {\r
+ updateTable(false);\r
+ }\r
+ };\r
+ worker.execute();\r
+ }\r
\r
/**\r
* Displays the create repository dialog and fires a SwingWorker to update\r
import org.eclipse.jgit.errors.StopWalkException;\r
import org.eclipse.jgit.lib.BlobBasedConfig;\r
import org.eclipse.jgit.lib.CommitBuilder;\r
-import org.eclipse.jgit.lib.Config;\r
import org.eclipse.jgit.lib.Constants;\r
import org.eclipse.jgit.lib.FileMode;\r
import org.eclipse.jgit.lib.ObjectId;\r
import org.eclipse.jgit.lib.RefUpdate.Result;\r
import org.eclipse.jgit.lib.Repository;\r
import org.eclipse.jgit.lib.RepositoryCache.FileKey;\r
-import org.eclipse.jgit.lib.StoredConfig;\r
import org.eclipse.jgit.lib.TreeFormatter;\r
import org.eclipse.jgit.revwalk.RevBlob;\r
import org.eclipse.jgit.revwalk.RevCommit;\r
return success;\r
}\r
\r
- /**\r
- * Returns a StoredConfig object for the repository.\r
- * \r
- * @param repository\r
- * @return the StoredConfig of the repository\r
- */\r
- public static StoredConfig readConfig(Repository repository) {\r
- StoredConfig c = repository.getConfig();\r
- try {\r
- c.load();\r
- } catch (ConfigInvalidException cex) {\r
- error(cex, repository, "{0} configuration is invalid!");\r
- } catch (IOException cex) {\r
- error(cex, repository, "Could not open configuration for {0}!");\r
- }\r
- return c;\r
- }\r
-\r
/**\r
* Zips the contents of the tree at the (optionally) specified revision and\r
* the (optionally) specified basepath to the supplied outputstream.\r
}\r
return null;\r
}\r
+ \r
+ public int size() {\r
+ return cache.size();\r
+ }\r
}\r
password);\r
\r
}\r
+ \r
+ /**\r
+ * Clears the repository cache on the Gitblit server.\r
+ * \r
+ * @param serverUrl\r
+ * @param account\r
+ * @param password\r
+ * @return true if the action succeeded\r
+ * @throws IOException\r
+ */\r
+ public static boolean clearRepositoryCache(String serverUrl, String account, \r
+ char[] password) throws IOException {\r
+ return doAction(RpcRequest.CLEAR_REPOSITORY_CACHE, null, null, serverUrl, account,\r
+ password);\r
+ }\r
\r
/**\r
* Create a user on the Gitblit server.\r
gb.authorizationControl = authorization control\r
gb.allowAuthenticatedDescription = grant restricted access to all authenticated users\r
gb.allowNamedDescription = grant restricted access to named users or teams\r
-gb.markdownFailure = Failed to parse Markdown content!
\ No newline at end of file
+gb.markdownFailure = Failed to parse Markdown content!\r
+gb.clearCache = clear cache
\ No newline at end of file
<wicket:fragment wicket:id="adminLinks">\r
<!-- page nav links --> \r
<div class="admin_nav">\r
- <img style="vertical-align: middle;" src="add_16x16.png"/>\r
- <a wicket:id="newRepository">\r
+ <a class="btn-small" wicket:id="clearCache">\r
+ <i class="icon icon-remove"></i>\r
+ <wicket:message key="gb.clearCache"></wicket:message>\r
+ </a> \r
+ <a class="btn-small" wicket:id="newRepository" style="padding-right:0px;">\r
+ <i class="icon icon-plus-sign"></i>\r
<wicket:message key="gb.newRepository"></wicket:message>\r
</a>\r
</div> \r
import com.gitblit.wicket.pages.BasePage;\r
import com.gitblit.wicket.pages.EditRepositoryPage;\r
import com.gitblit.wicket.pages.EmptyRepositoryPage;\r
+import com.gitblit.wicket.pages.RepositoriesPage;\r
import com.gitblit.wicket.pages.SummaryPage;\r
\r
public class RepositoriesPanel extends BasePanel {\r
final IDataProvider<RepositoryModel> dp;\r
\r
Fragment adminLinks = new Fragment("adminPanel", "adminLinks", this);\r
+ adminLinks.add(new Link<Void>("clearCache") {\r
+\r
+ private static final long serialVersionUID = 1L;\r
+\r
+ @Override\r
+ public void onClick() {\r
+ GitBlit.self().resetRepositoryListCache();\r
+ setResponsePage(RepositoriesPage.class);\r
+ }\r
+ }.setVisible(GitBlit.getBoolean(Keys.git.cacheRepositoryList, true)));\r
adminLinks.add(new BookmarkablePageLink<Void>("newRepository", EditRepositoryPage.class));\r
add(adminLinks.setVisible(showAdmin));\r
\r
<wicket:fragment wicket:id="adminLinks">\r
<!-- page nav links --> \r
<div class="admin_nav">\r
- <img style="vertical-align: middle;" src="add_16x16.png"/>\r
- <a wicket:id="newTeam">\r
+ <a class="btn-small" wicket:id="newTeam" style="padding-right:0px;">\r
+ <i class="icon icon-plus-sign"></i>\r
<wicket:message key="gb.newTeam"></wicket:message>\r
</a>\r
</div> \r
<wicket:fragment wicket:id="adminLinks">\r
<!-- page nav links --> \r
<div class="admin_nav">\r
- <img style="vertical-align: middle;" src="add_16x16.png"/>\r
- <a wicket:id="newUser">\r
+ <a class="btn-small" wicket:id="newUser" style="padding-right:0px;">\r
+ <i class="icon icon-plus-sign"></i>\r
<wicket:message key="gb.newUser"></wicket:message>\r
</a>\r
</div> \r
import com.gitblit.Constants;
import com.gitblit.GitBlit;
import com.gitblit.models.RepositoryModel;
-import com.gitblit.utils.JGitUtils;
public class RepositoryModelTest {
@Before
public void initializeConfiguration() throws Exception{
Repository r = GitBlitSuite.getHelloworldRepository();
- StoredConfig config = JGitUtils.readConfig(r);
+ StoredConfig config = r.getConfig();
config.unsetSection(Constants.CONFIG_GITBLIT, Constants.CONFIG_CUSTOM_FIELDS);
config.setString(Constants.CONFIG_GITBLIT, Constants.CONFIG_CUSTOM_FIELDS, "commitMessageRegEx", "\\d");
@After
public void teardownConfiguration() throws Exception {
Repository r = GitBlitSuite.getHelloworldRepository();
- StoredConfig config = JGitUtils.readConfig(r);
+ StoredConfig config = r.getConfig();
config.unsetSection(Constants.CONFIG_GITBLIT, Constants.CONFIG_CUSTOM_FIELDS);
config.save();
projects / gitblit.git / commitdiff