From 40ca5c51d1006cd1badf5393f20921b2047f4701 Mon Sep 17 00:00:00 2001 From: James Moger Date: Fri, 16 Mar 2012 17:01:19 -0400 Subject: [PATCH] Repository-branch Lucene indexing is now opt-in --- src/com/gitblit/GitBlit.java | 11 ++ src/com/gitblit/LuceneExecutor.java | 150 +++++++++--------- src/com/gitblit/models/RepositoryModel.java | 3 +- .../gitblit/wicket/GitBlitWebApp.properties | 7 +- .../wicket/pages/EditRepositoryPage.html | 2 + .../wicket/pages/EditRepositoryPage.java | 35 +++- src/com/gitblit/wicket/pages/LucenePage.java | 40 +++-- .../com/gitblit/tests/LuceneExecutorTest.java | 58 ++++--- 8 files changed, 191 insertions(+), 115 deletions(-) diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java index 15b69ec0..272630c1 100644 --- a/src/com/gitblit/GitBlit.java +++ b/src/com/gitblit/GitBlit.java @@ -812,6 +812,8 @@ public class GitBlit implements ServletContextListener { "gitblit", null, "postReceiveScript"))); model.mailingLists = new ArrayList(Arrays.asList(config.getStringList( "gitblit", null, "mailingList"))); + model.indexedBranches = new ArrayList(Arrays.asList(config.getStringList( + "gitblit", null, "indexBranch"))); } model.HEAD = JGitUtils.getHEADRef(r); model.availableRefs = JGitUtils.getAvailableHeadTargets(r); @@ -955,6 +957,12 @@ public class GitBlit implements ServletContextListener { // create repository logger.info("create repository " + repository.name); r = JGitUtils.createRepository(repositoriesFolder, repository.name); + + // automatically index master branch if Lucene integration is enabled + if (luceneExecutor.isReady()) { + repository.indexedBranches = new ArrayList(); + repository.indexedBranches.add("refs/heads/master"); + } } else { // rename repository if (!repositoryName.equalsIgnoreCase(repository.name)) { @@ -1063,6 +1071,9 @@ public class GitBlit implements ServletContextListener { if (!ArrayUtils.isEmpty(repository.mailingLists)) { config.setStringList("gitblit", null, "mailingList", repository.mailingLists); } + if (!ArrayUtils.isEmpty(repository.indexedBranches)) { + config.setStringList("gitblit", null, "indexBranch", repository.indexedBranches); + } try { config.save(); } catch (IOException e) { diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java index 0b90b74c..e4ee0b67 100644 --- a/src/com/gitblit/LuceneExecutor.java +++ b/src/com/gitblit/LuceneExecutor.java @@ -89,6 +89,7 @@ import com.gitblit.models.IssueModel; import com.gitblit.models.IssueModel.Attachment; import com.gitblit.models.PathModel.PathChangeModel; import com.gitblit.models.RefModel; +import com.gitblit.models.RepositoryModel; import com.gitblit.models.SearchResult; import com.gitblit.utils.ArrayUtils; import com.gitblit.utils.IssueUtils; @@ -141,9 +142,6 @@ public class LuceneExecutor implements Runnable { "arj", "bin", "bmp", "dll", "doc", "docx", "exe", "gif", "gz", "jar", "jpg", "lib", "lzh", "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls", "xlsx", "zip")); - private final Set excludedBranches = new TreeSet( - Arrays.asList("/refs/heads/gb-issues")); - public LuceneExecutor(IStoredSettings settings, File repositoriesFolder) { this.storedSettings = settings; this.repositoriesFolder = repositoriesFolder; @@ -170,18 +168,14 @@ public class LuceneExecutor implements Runnable { return; } - for (String repositoryName : GitBlit.self().getRepositoryList()) { - Repository repository = GitBlit.self().getRepository(repositoryName); - if (repository == null) { - logger.warn(MessageFormat.format( - "Lucene executor could not find repository {0}. Skipping.", - repositoryName)); - continue; + for (String repositoryName: GitBlit.self().getRepositoryList()) { + RepositoryModel model = GitBlit.self().getRepositoryModel(repositoryName); + if (model.hasCommits && !ArrayUtils.isEmpty(model.indexedBranches)) { + Repository repository = GitBlit.self().getRepository(model.name); + index(model, repository); + repository.close(); + System.gc(); } - // TODO allow repository to bypass Lucene indexing - index(repositoryName, repository); - repository.close(); - System.gc(); } } @@ -194,43 +188,38 @@ public class LuceneExecutor implements Runnable { * @param repository * the repository object */ - protected void index(String name, Repository repository) { + protected void index(RepositoryModel model, Repository repository) { try { - if (JGitUtils.hasCommits(repository)) { - if (shouldReindex(repository)) { - // (re)build the entire index - IndexResult result = reindex(name, repository); - - if (result.success) { - if (result.commitCount > 0) { - String msg = "Built {0} Lucene index from {1} commits and {2} files across {3} branches in {4} secs"; - logger.info(MessageFormat.format(msg, name, - result.commitCount, result.blobCount, result.branchCount, result.duration())); - } - } else { - String msg = "Could not build {0} Lucene index!"; - logger.error(MessageFormat.format(msg, name)); + if (shouldReindex(repository)) { + // (re)build the entire index + IndexResult result = reindex(model, repository); + + if (result.success) { + if (result.commitCount > 0) { + String msg = "Built {0} Lucene index from {1} commits and {2} files across {3} branches in {4} secs"; + logger.info(MessageFormat.format(msg, model.name, result.commitCount, + result.blobCount, result.branchCount, result.duration())); } } else { - // update the index with latest commits - IndexResult result = updateIndex(name, repository); - if (result.success) { - if (result.commitCount > 0) { - String msg = "Updated {0} Lucene index with {1} commits and {2} files across {3} branches in {4} secs"; - logger.info(MessageFormat.format(msg, name, - result.commitCount, result.blobCount, result.branchCount, result.duration())); - } - } else { - String msg = "Could not update {0} Lucene index!"; - logger.error(MessageFormat.format(msg, name)); - } + String msg = "Could not build {0} Lucene index!"; + logger.error(MessageFormat.format(msg, model.name)); } } else { - logger.info(MessageFormat.format("Skipped Lucene index of empty repository {0}", - name)); + // update the index with latest commits + IndexResult result = updateIndex(model, repository); + if (result.success) { + if (result.commitCount > 0) { + String msg = "Updated {0} Lucene index with {1} commits and {2} files across {3} branches in {4} secs"; + logger.info(MessageFormat.format(msg, model.name, result.commitCount, + result.blobCount, result.branchCount, result.duration())); + } + } else { + String msg = "Could not update {0} Lucene index!"; + logger.error(MessageFormat.format(msg, model.name)); + } } } catch (Throwable t) { - logger.error(MessageFormat.format("Lucene indexing failure for {0}", name), t); + logger.error(MessageFormat.format("Lucene indexing failure for {0}", model.name), t); } } @@ -430,15 +419,15 @@ public class LuceneExecutor implements Runnable { * @param repository * @return IndexResult */ - public IndexResult reindex(String repositoryName, Repository repository) { + public IndexResult reindex(RepositoryModel model, Repository repository) { IndexResult result = new IndexResult(); - if (!deleteIndex(repositoryName)) { + if (!deleteIndex(model.name)) { return result; } try { FileBasedConfig config = getConfig(repository); Set indexedCommits = new TreeSet(); - IndexWriter writer = getIndexWriter(repositoryName); + IndexWriter writer = getIndexWriter(model.name); // build a quick lookup of tags Map> tags = new HashMap>(); for (RefModel tag : JGitUtils.getTags(repository, false, -1)) { @@ -479,7 +468,9 @@ public class LuceneExecutor implements Runnable { // walk through each branch for (RefModel branch : branches) { - if (excludedBranches.contains(branch.getName())) { + + // if this branch is not specifically indexed then skip + if (!model.indexedBranches.contains(branch.getName())) { continue; } @@ -624,11 +615,11 @@ public class LuceneExecutor implements Runnable { // commit all changes and reset the searcher config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION); config.save(); - resetIndexSearcher(repositoryName); + resetIndexSearcher(model.name); writer.commit(); result.success(); } catch (Exception e) { - logger.error("Exception while reindexing " + repositoryName, e); + logger.error("Exception while reindexing " + model.name, e); } return result; } @@ -648,24 +639,6 @@ public class LuceneExecutor implements Runnable { String branch, RevCommit commit) { IndexResult result = new IndexResult(); try { - if (excludedBranches.contains(branch)) { - if (IssueUtils.GB_ISSUES.equals(branch)) { - // index an issue - String issueId = commit.getShortMessage().substring(2).trim(); - IssueModel issue = IssueUtils.getIssue(repository, issueId); - if (issue == null) { - // issue was deleted, remove from index - deleteIssue(repositoryName, issueId); - result.success = true; - return result; - } - result.success = index(repositoryName, issue); - result.issueCount++; - return result; - - } - return result; - } List changedPaths = JGitUtils.getFilesInCommit(repository, commit); String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L, Resolution.MINUTE); @@ -779,11 +752,11 @@ public class LuceneExecutor implements Runnable { /** * Updates a repository index incrementally from the last indexed commits. * - * @param repositoryName + * @param model * @param repository * @return IndexResult */ - protected IndexResult updateIndex(String repositoryName, Repository repository) { + protected IndexResult updateIndex(RepositoryModel model, Repository repository) { IndexResult result = new IndexResult(); try { FileBasedConfig config = getConfig(repository); @@ -816,6 +789,12 @@ public class LuceneExecutor implements Runnable { for (RefModel branch : branches) { String branchName = branch.getName(); + // determine if we should skip this branch + if (!IssueUtils.GB_ISSUES.equals(branch) + && !model.indexedBranches.contains(branch.getName())) { + continue; + } + // remove this branch from the deletedBranches set deletedBranches.remove(branchName); @@ -836,10 +815,33 @@ public class LuceneExecutor implements Runnable { result.branchCount += 1; } + // track the issue ids that we have already indexed + Set indexedIssues = new TreeSet(); + // reverse the list of commits so we start with the first commit Collections.reverse(revs); - for (RevCommit commit : revs) { - result.add(index(repositoryName, repository, branchName, commit)); + for (RevCommit commit : revs) { + if (IssueUtils.GB_ISSUES.equals(branch)) { + // only index an issue once during updateIndex + String issueId = commit.getShortMessage().substring(2).trim(); + if (indexedIssues.contains(issueId)) { + continue; + } + indexedIssues.add(issueId); + + IssueModel issue = IssueUtils.getIssue(repository, issueId); + if (issue == null) { + // issue was deleted, remove from index + deleteIssue(model.name, issueId); + } else { + // issue was updated + index(model.name, issue); + result.issueCount++; + } + } else { + // index a commit + result.add(index(model.name, repository, branchName, commit)); + } } // update the config @@ -853,14 +855,14 @@ public class LuceneExecutor implements Runnable { // unless a branch really was deleted and no longer exists if (deletedBranches.size() > 0) { for (String branch : deletedBranches) { - IndexWriter writer = getIndexWriter(repositoryName); + IndexWriter writer = getIndexWriter(model.name); writer.deleteDocuments(new Term(FIELD_BRANCH, branch)); writer.commit(); } } result.success = true; } catch (Throwable t) { - logger.error(MessageFormat.format("Exception while updating {0} Lucene index", repositoryName), t); + logger.error(MessageFormat.format("Exception while updating {0} Lucene index", model.name), t); } return result; } diff --git a/src/com/gitblit/models/RepositoryModel.java b/src/com/gitblit/models/RepositoryModel.java index 10dcbc68..c087a358 100644 --- a/src/com/gitblit/models/RepositoryModel.java +++ b/src/com/gitblit/models/RepositoryModel.java @@ -57,6 +57,7 @@ public class RepositoryModel implements Serializable, Comparable availableRefs; + public List indexedBranches; public String size; public List preReceiveScripts; public List postReceiveScripts; @@ -74,7 +75,7 @@ public class RepositoryModel implements Serializable, Comparable(); - this.federationStrategy = FederationStrategy.FEDERATE_THIS; + this.federationStrategy = FederationStrategy.FEDERATE_THIS; } @Override diff --git a/src/com/gitblit/wicket/GitBlitWebApp.properties b/src/com/gitblit/wicket/GitBlitWebApp.properties index 9ce4df5d..0952b573 100644 --- a/src/com/gitblit/wicket/GitBlitWebApp.properties +++ b/src/com/gitblit/wicket/GitBlitWebApp.properties @@ -216,4 +216,9 @@ gb.workingCopyWarning = this repository has a working copy and can not receive p gb.query = query gb.queryHelp = write brief explanation of Lucene syntax here gb.authored = authored -gb.committed = committed \ No newline at end of file +gb.committed = committed +gb.indexedBranches = indexed branches +gb.indexedBranchesDescription = select the branches to include in your Lucene index +gb.noIndexedRepositoriesWarning = none of your repositories are configured for Lucene indexing +gb.undefinedQueryWarning = query is undefined! +gb.gb.noSelectedRepositoriesWarning = please select one or more repositories! \ No newline at end of file diff --git a/src/com/gitblit/wicket/pages/EditRepositoryPage.html b/src/com/gitblit/wicket/pages/EditRepositoryPage.html index 2a1ae70c..26ef37c8 100644 --- a/src/com/gitblit/wicket/pages/EditRepositoryPage.html +++ b/src/com/gitblit/wicket/pages/EditRepositoryPage.html @@ -31,6 +31,8 @@