From 43e32ec71a508e1bb68b247fdca74f64bcf629b3 Mon Sep 17 00:00:00 2001 From: James Moger Date: Tue, 28 Feb 2012 22:41:31 -0500 Subject: [PATCH] Draft integration of Lucene search mechanism and web ui --- distrib/gitblit.properties | 20 ++++++++++ resources/gitblit.css | 12 ++++++ src/com/gitblit/GitBlit.java | 4 +- src/com/gitblit/LuceneExecutor.java | 27 ++++++++----- src/com/gitblit/utils/LuceneUtils.java | 44 ++++++++++++++-------- src/com/gitblit/wicket/pages/RootPage.java | 3 ++ 6 files changed, 83 insertions(+), 27 deletions(-) diff --git a/distrib/gitblit.properties b/distrib/gitblit.properties index 2ac6598f..ede5f596 100644 --- a/distrib/gitblit.properties +++ b/distrib/gitblit.properties @@ -91,6 +91,26 @@ groovy.preReceiveScripts = # SINCE 0.8.0 groovy.postReceiveScripts = +# If true, a Lucene index will be generated and maintained for each repository. +# Lucene search replaces brute-force Git repository traversal. +# +# SINCE 0.9.0 +# RESTART REQUIRED +lucene.enable = false + +# If *lucene.pollingMode* = true, Gitblit will periodically check all repositories +# for branch updates. +# If *lucene.pollingMode* = false, repositories will only be indexed on pushes +# to Gitblit. +# +# Regardless of this setting, Gitblit will check all repositories for branch +# updates 1 minute after startup. Indexes will automatically be built for any +# repository that is missing its index or if an index version change is detected. +# +# SINCE 0.9.0 +# RESTART REQUIRED +lucene.pollingMode = false + # # Authentication Settings # diff --git a/resources/gitblit.css b/resources/gitblit.css index 6ad45fb2..bfa1c202 100644 --- a/resources/gitblit.css +++ b/resources/gitblit.css @@ -266,6 +266,18 @@ img.gravatar { padding: 2px; } +div.searchResult { + padding:5px; +} + +div.searchResult .summary { + font-weight: bold; +} + +div.searchResult .branch { + color: #008000; +} + div.header, div.commitHeader, table.repositories th { background-color:#e0e0e0; background-repeat:repeat-x; diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java index a7bfad4f..f6691dc6 100644 --- a/src/com/gitblit/GitBlit.java +++ b/src/com/gitblit/GitBlit.java @@ -1824,8 +1824,8 @@ public class GitBlit implements ServletContextListener { } luceneExecutor = new LuceneExecutor(settings); if (luceneExecutor.isReady()) { - logger.info("Lucene executor is scheduled to process the repository queue every 10 minutes."); - scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 10, TimeUnit.MINUTES); + logger.info("Lucene executor is scheduled to process the repository queue every 2 minutes."); + scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 2, TimeUnit.MINUTES); } else { logger.warn("Lucene executor is disabled."); } diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java index e4624efc..b4e5134a 100644 --- a/src/com/gitblit/LuceneExecutor.java +++ b/src/com/gitblit/LuceneExecutor.java @@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory; import com.gitblit.models.RepositoryModel; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.LuceneUtils; +import com.gitblit.utils.LuceneUtils.IndexResult; /** * The Lucene executor handles indexing repositories synchronously and @@ -53,8 +54,8 @@ public class LuceneExecutor implements Runnable { public LuceneExecutor(IStoredSettings settings) { this.settings = settings; - this.isLuceneEnabled = settings.getBoolean("lucene.enableLucene", false); - this.isPollingMode = settings.getBoolean("lucene.pollingMode", false); + this.isLuceneEnabled = settings.getBoolean(Keys.lucene.enable, false); + this.isPollingMode = settings.getBoolean(Keys.lucene.pollingMode, false); } /** @@ -144,11 +145,14 @@ public class LuceneExecutor implements Runnable { if (LuceneUtils.shouldReindex(repository)) { // (re)build the entire index long start = System.currentTimeMillis(); - boolean success = LuceneUtils.reindex(repository); + IndexResult result = LuceneUtils.reindex(repository); long duration = System.currentTimeMillis() - start; - if (success) { - String msg = "Built {0} Lucene index in {1} msecs"; - logger.info(MessageFormat.format(msg, repositoryName, duration)); + if (result.success) { + if (result.commitCount > 0) { + String msg = "Built {0} Lucene index from {1} commits in {2} msecs"; + logger.info(MessageFormat.format(msg, repositoryName, + result.commitCount, duration)); + } } else { String msg = "Could not build {0} Lucene index!"; logger.error(MessageFormat.format(msg, repositoryName)); @@ -156,11 +160,14 @@ public class LuceneExecutor implements Runnable { } else { // update the index with latest commits long start = System.currentTimeMillis(); - boolean success = LuceneUtils.updateIndex(repository); + IndexResult result = LuceneUtils.updateIndex(repository); long duration = System.currentTimeMillis() - start; - if (success) { - String msg = "Updated {0} Lucene index in {1} msecs"; - logger.info(MessageFormat.format(msg, repositoryName, duration)); + if (result.success) { + if (result.commitCount > 0) { + String msg = "Updated {0} Lucene index with {1} commits in {2} msecs"; + logger.info(MessageFormat.format(msg, repositoryName, + result.commitCount, duration)); + } } else { String msg = "Could not update {0} Lucene index!"; logger.error(MessageFormat.format(msg, repositoryName)); diff --git a/src/com/gitblit/utils/LuceneUtils.java b/src/com/gitblit/utils/LuceneUtils.java index d463cdf1..e8242362 100644 --- a/src/com/gitblit/utils/LuceneUtils.java +++ b/src/com/gitblit/utils/LuceneUtils.java @@ -52,6 +52,7 @@ import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.util.FS; +import com.gitblit.GitBlit; import com.gitblit.models.IssueModel; import com.gitblit.models.IssueModel.Attachment; import com.gitblit.models.PathModel.PathChangeModel; @@ -121,10 +122,13 @@ public class LuceneUtils { * @return the repository name */ private static String getName(Repository repository) { + String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath(); if (repository.isBare()) { - return repository.getDirectory().getName(); + return StringUtils.getRelativePath(rootPath, repository.getDirectory() + .getAbsolutePath()); } else { - return repository.getDirectory().getParentFile().getName(); + return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile() + .getAbsolutePath()); } } @@ -198,11 +202,12 @@ public class LuceneUtils { * index. * * @param repository - * @return true if the indexing has succeeded + * @return IndexResult */ - public static boolean reindex(Repository repository) { + public static IndexResult reindex(Repository repository) { + IndexResult result = new IndexResult(); if (!LuceneUtils.deleteIndex(repository)) { - return false; + return result; } try { String repositoryName = getName(repository); @@ -300,6 +305,7 @@ public class LuceneUtils { Index.NOT_ANALYZED)); doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED)); writer.addDocument(doc); + result.commitCount += 1; } // traverse the log and index the previous commit objects @@ -312,6 +318,7 @@ public class LuceneUtils { Index.NOT_ANALYZED)); doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED)); writer.addDocument(doc); + result.commitCount += 1; } } @@ -335,11 +342,11 @@ public class LuceneUtils { config.save(); resetIndexSearcher(repository); writer.commit(); - return true; + result.success = true; } catch (Exception e) { e.printStackTrace(); } - return false; + return result; } /** @@ -453,9 +460,10 @@ public class LuceneUtils { * Updates a repository index incrementally from the last indexed commits. * * @param repository + * @return IndexResult */ - public static boolean updateIndex(Repository repository) { - boolean success = false; + public static IndexResult updateIndex(Repository repository) { + IndexResult result = new IndexResult(); try { FileBasedConfig config = getConfig(repository); config.load(); @@ -475,13 +483,13 @@ public class LuceneUtils { // detect branch deletion // first assume all branches are deleted and then remove each - // existing branch from deletedBranches during indexing + // existing branch from deletedBranches during indexing Set deletedBranches = new TreeSet(); for (String alias : config.getNames(CONF_ALIAS)) { String branch = config.getString(CONF_ALIAS, null, alias); deletedBranches.add(branch); } - + // walk through each branches List branches = JGitUtils.getLocalBranches(repository, true, -1); for (RefModel branch : branches) { @@ -490,7 +498,7 @@ public class LuceneUtils { // remove this branch from the deletedBranches set deletedBranches.remove(branchName); - // determine last commit + // determine last commit String keyName = getBranchKey(branchName); String lastCommit = config.getString(CONF_BRANCH, null, keyName); @@ -507,6 +515,7 @@ public class LuceneUtils { Collections.reverse(revs); for (RevCommit commit : revs) { index(repository, branchName, commit); + result.commitCount += 1; } // update the config @@ -515,7 +524,7 @@ public class LuceneUtils { config.setString(CONF_BRANCH, null, keyName, branch.getObjectId().getName()); config.save(); } - + // the deletedBranches set will normally be empty by this point // unless a branch really was deleted and no longer exists if (deletedBranches.size() > 0) { @@ -525,11 +534,11 @@ public class LuceneUtils { writer.commit(); } } - success = true; + result.success = true; } catch (Throwable t) { t.printStackTrace(); } - return success; + return result; } /** @@ -782,4 +791,9 @@ public class LuceneUtils { } SEARCHERS.clear(); } + + public static class IndexResult { + public boolean success; + public int commitCount; + } } diff --git a/src/com/gitblit/wicket/pages/RootPage.java b/src/com/gitblit/wicket/pages/RootPage.java index bad0140b..d81fae1b 100644 --- a/src/com/gitblit/wicket/pages/RootPage.java +++ b/src/com/gitblit/wicket/pages/RootPage.java @@ -101,6 +101,9 @@ public abstract class RootPage extends BasePage { pages.add(new PageRegistration("gb.repositories", RepositoriesPage.class, getRootPageParameters())); pages.add(new PageRegistration("gb.activity", ActivityPage.class, getRootPageParameters())); + if (GitBlit.getBoolean(Keys.lucene.enable, false)) { + pages.add(new PageRegistration("gb.search", LucenePage.class)); + } if (showAdmin) { pages.add(new PageRegistration("gb.users", UsersPage.class)); } -- 2.39.5