From 3d0494ad6db80843add9bf9d6eac783a691ac5a1 Mon Sep 17 00:00:00 2001 From: James Moger Date: Fri, 9 Mar 2012 20:45:52 -0500 Subject: [PATCH] More Lucene progress --- src/com/gitblit/GitBlit.java | 1 + src/com/gitblit/LuceneExecutor.java | 52 ++-- src/com/gitblit/models/SearchResult.java | 2 + src/com/gitblit/utils/LuceneUtils.java | 261 ++++++++++++------ src/com/gitblit/wicket/GitBlitWebApp.java | 22 +- .../gitblit/wicket/GitBlitWebApp.properties | 4 +- src/com/gitblit/wicket/pages/LucenePage.html | 37 ++- src/com/gitblit/wicket/pages/LucenePage.java | 80 +++++- .../gitblit/wicket/pages/RepositoryPage.java | 12 +- .../gitblit/wicket/panels/ActivityPanel.java | 8 +- .../gitblit/wicket/panels/BranchesPanel.java | 8 +- .../gitblit/wicket/panels/HistoryPanel.java | 8 +- src/com/gitblit/wicket/panels/LogPanel.java | 8 +- tests/com/gitblit/tests/IssuesTest.java | 7 +- tests/com/gitblit/tests/LuceneUtilsTest.java | 17 +- 15 files changed, 384 insertions(+), 143 deletions(-) diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java index f6691dc6..e2240256 100644 --- a/src/com/gitblit/GitBlit.java +++ b/src/com/gitblit/GitBlit.java @@ -184,6 +184,7 @@ public class GitBlit implements ServletContextListener { } return self().timezone; } + /** * Returns the boolean value for the specified key. If the key does not diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java index b4e5134a..4f06b4ec 100644 --- a/src/com/gitblit/LuceneExecutor.java +++ b/src/com/gitblit/LuceneExecutor.java @@ -105,26 +105,26 @@ public class LuceneExecutor implements Runnable { Set processed = new HashSet(); if (!queue.isEmpty()) { // update the repository Lucene index - String repositoryName = null; - while ((repositoryName = queue.poll()) != null) { - if (processed.contains(repositoryName)) { + String name = null; + while ((name = queue.poll()) != null) { + if (processed.contains(name)) { // skipping multi-queued repository continue; } try { - Repository repository = GitBlit.self().getRepository(repositoryName); + Repository repository = GitBlit.self().getRepository(name); if (repository == null) { logger.warn(MessageFormat.format( "Lucene executor could not find repository {0}. Skipping.", - repositoryName)); + name)); continue; } - index(repositoryName, repository); + index(name, repository); repository.close(); - processed.add(repositoryName); + processed.add(name); } catch (Throwable e) { logger.error(MessageFormat.format("Failed to update {0} Lucene index", - repositoryName), e); + name), e); } } } @@ -134,51 +134,53 @@ public class LuceneExecutor implements Runnable { * Synchronously indexes a repository. This may build a complete index of a * repository or it may update an existing index. * - * @param repositoryName + * @param name * the name of the repository * @param repository * the repository object */ - public void index(String repositoryName, Repository repository) { + public void index(String name, Repository repository) { try { if (JGitUtils.hasCommits(repository)) { if (LuceneUtils.shouldReindex(repository)) { // (re)build the entire index long start = System.currentTimeMillis(); - IndexResult result = LuceneUtils.reindex(repository); - long duration = System.currentTimeMillis() - start; + String msg = "Building {0} Lucene index..."; + logger.info(MessageFormat.format(msg, name)); + IndexResult result = LuceneUtils.reindex(name, repository, true); + float duration = (System.currentTimeMillis() - start)/1000f; if (result.success) { if (result.commitCount > 0) { - String msg = "Built {0} Lucene index from {1} commits in {2} msecs"; - logger.info(MessageFormat.format(msg, repositoryName, - result.commitCount, duration)); + msg = "Built {0} Lucene index from {1} commits and {2} files across {3} branches in {4} secs"; + logger.info(MessageFormat.format(msg, name, + result.commitCount, result.blobCount, result.branchCount, duration)); } } else { - String msg = "Could not build {0} Lucene index!"; - logger.error(MessageFormat.format(msg, repositoryName)); + msg = "Could not build {0} Lucene index!"; + logger.error(MessageFormat.format(msg, name)); } } else { // update the index with latest commits long start = System.currentTimeMillis(); - IndexResult result = LuceneUtils.updateIndex(repository); - long duration = System.currentTimeMillis() - start; + IndexResult result = LuceneUtils.updateIndex(name, repository); + float duration = (System.currentTimeMillis() - start)/1000f; if (result.success) { if (result.commitCount > 0) { - String msg = "Updated {0} Lucene index with {1} commits in {2} msecs"; - logger.info(MessageFormat.format(msg, repositoryName, - result.commitCount, duration)); + String msg = "Updated {0} Lucene index with {1} commits and {2} files across {3} branches in {4} secs"; + logger.info(MessageFormat.format(msg, name, + result.commitCount, result.blobCount, result.branchCount, duration)); } } else { String msg = "Could not update {0} Lucene index!"; - logger.error(MessageFormat.format(msg, repositoryName)); + logger.error(MessageFormat.format(msg, name)); } } } else { logger.info(MessageFormat.format("Skipped Lucene index of empty repository {0}", - repositoryName)); + name)); } } catch (Throwable t) { - logger.error(MessageFormat.format("Lucene indexing failure for {0}", repositoryName), t); + logger.error(MessageFormat.format("Lucene indexing failure for {0}", name), t); } } diff --git a/src/com/gitblit/models/SearchResult.java b/src/com/gitblit/models/SearchResult.java index 2fa0db42..ffe4d870 100644 --- a/src/com/gitblit/models/SearchResult.java +++ b/src/com/gitblit/models/SearchResult.java @@ -32,6 +32,8 @@ public class SearchResult implements Serializable { public String id; + public List tags; + public List labels; public ObjectType type; diff --git a/src/com/gitblit/utils/LuceneUtils.java b/src/com/gitblit/utils/LuceneUtils.java index e8242362..fe36a838 100644 --- a/src/com/gitblit/utils/LuceneUtils.java +++ b/src/com/gitblit/utils/LuceneUtils.java @@ -1,3 +1,18 @@ +/* + * Copyright 2012 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitblit.utils; import java.io.ByteArrayOutputStream; @@ -50,9 +65,11 @@ import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.AndTreeFilter; +import org.eclipse.jgit.treewalk.filter.PathFilterGroup; +import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.FS; -import com.gitblit.GitBlit; import com.gitblit.models.IssueModel; import com.gitblit.models.IssueModel.Attachment; import com.gitblit.models.PathModel.PathChangeModel; @@ -96,6 +113,7 @@ public class LuceneUtils { private static final String FIELD_AUTHOR = "author"; private static final String FIELD_COMMITTER = "committer"; private static final String FIELD_DATE = "date"; + private static final String FIELD_TAG = "tag"; private static final String FIELD_LABEL = "label"; private static final String FIELD_ATTACHMENT = "attachment"; @@ -109,27 +127,47 @@ public class LuceneUtils { private static final Map SEARCHERS = new ConcurrentHashMap(); private static final Map WRITERS = new ConcurrentHashMap(); + private static final String LUCENE_DIR = "lucene"; private static final String CONF_FILE = "lucene.conf"; private static final String CONF_INDEX = "index"; private static final String CONF_VERSION = "version"; private static final String CONF_ALIAS = "aliases"; private static final String CONF_BRANCH = "branches"; - + /** - * Returns the name of the repository. + * Returns the author for the commit, if this information is available. * - * @param repository - * @return the repository name + * @param commit + * @return an author or unknown */ - private static String getName(Repository repository) { - String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath(); - if (repository.isBare()) { - return StringUtils.getRelativePath(rootPath, repository.getDirectory() - .getAbsolutePath()); - } else { - return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile() - .getAbsolutePath()); + private static String getAuthor(RevCommit commit) { + String name = "unknown"; + try { + name = commit.getAuthorIdent().getName(); + if (StringUtils.isEmpty(name)) { + name = commit.getAuthorIdent().getEmailAddress(); + } + } catch (NullPointerException n) { } + return name; + } + + /** + * Returns the committer for the commit, if this information is available. + * + * @param commit + * @return an committer or unknown + */ + private static String getCommitter(RevCommit commit) { + String name = "unknown"; + try { + name = commit.getCommitterIdent().getName(); + if (StringUtils.isEmpty(name)) { + name = commit.getCommitterIdent().getEmailAddress(); + } + } catch (NullPointerException n) { + } + return name; } /** @@ -182,7 +220,7 @@ public class LuceneUtils { */ public static boolean deleteIndex(Repository repository) { try { - File luceneIndex = new File(repository.getDirectory(), "lucene"); + File luceneIndex = new File(repository.getDirectory(), LUCENE_DIR); if (luceneIndex.exists()) { org.eclipse.jgit.util.FileUtils.delete(luceneIndex, org.eclipse.jgit.util.FileUtils.RECURSIVE); @@ -201,16 +239,22 @@ public class LuceneUtils { * This completely indexes the repository and will destroy any existing * index. * + * @param repositoryName * @param repository + * @param fullIndex + * If false blob metadata is set to the HEAD revision of each + * branch. If true, each the last commit of each blob is + * determined to properly index the author, committer, and date. + * Full indexing can be time-consuming. * @return IndexResult */ - public static IndexResult reindex(Repository repository) { + public static IndexResult reindex(String repositoryName, Repository repository, + boolean fullIndex) { IndexResult result = new IndexResult(); if (!LuceneUtils.deleteIndex(repository)) { return result; } - try { - String repositoryName = getName(repository); + try { FileBasedConfig config = getConfig(repository); Set indexedCommits = new TreeSet(); IndexWriter writer = getIndexWriter(repository, true); @@ -235,40 +279,64 @@ public class LuceneUtils { } String branchName = branch.getName(); RevWalk revWalk = new RevWalk(repository); - RevCommit rev = revWalk.parseCommit(branch.getObjectId()); + RevCommit branchHead = revWalk.parseCommit(branch.getObjectId()); + String head = branchHead.getId().getName(); String keyName = getBranchKey(branchName); config.setString(CONF_ALIAS, null, keyName, branchName); - config.setString(CONF_BRANCH, null, keyName, rev.getName()); + config.setString(CONF_BRANCH, null, keyName, head); // index the blob contents of the tree ByteArrayOutputStream os = new ByteArrayOutputStream(); byte[] tmp = new byte[32767]; TreeWalk treeWalk = new TreeWalk(repository); - treeWalk.addTree(rev.getTree()); + treeWalk.addTree(branchHead.getTree()); treeWalk.setRecursive(true); - String revDate = DateTools.timeToString(rev.getCommitTime() * 1000L, - Resolution.MINUTE); + while (treeWalk.next()) { + result.blobCount++; + String blobPath = treeWalk.getPathString(); + RevCommit blobRev = branchHead; + + RevWalk blobWalk = null; + if (fullIndex) { + // XXX this is _really_ slow, there must be a better way + // determine the most recent commit for this blob + blobWalk = new RevWalk(repository); + blobWalk.markStart(blobWalk.parseCommit(branch.getObjectId())); + TreeFilter filter = AndTreeFilter.create( + PathFilterGroup.createFromStrings(Collections.singleton(blobPath)), + TreeFilter.ANY_DIFF); + blobWalk.setTreeFilter(filter); + + for (RevCommit commit : blobWalk) { + blobRev = commit; + break; + } + } + + String blobAuthor = getAuthor(blobRev); + String blobCommitter = getCommitter(blobRev); + String blobDate = DateTools.timeToString(blobRev.getCommitTime() * 1000L, + Resolution.MINUTE); + + if (blobWalk != null) { + blobWalk.dispose(); + } + Document doc = new Document(); - doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES, - Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, - Index.NOT_ANALYZED)); - doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED)); - doc.add(new Field(FIELD_OBJECT_ID, treeWalk.getPathString(), Store.YES, - Index.NOT_ANALYZED)); - doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO)); - doc.add(new Field(FIELD_AUTHOR, rev.getAuthorIdent().getName(), Store.YES, - Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field(FIELD_COMMITTER, rev.getCommitterIdent().getName(), - Store.YES, Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field(FIELD_LABEL, branch.getName(), Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES, Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_OBJECT_ID, blobPath, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_DATE, blobDate, Store.YES, Index.NO)); + doc.add(new Field(FIELD_AUTHOR, blobAuthor, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_COMMITTER, blobCommitter, Store.YES, Index.ANALYZED)); // determine extension to compare to the extension // blacklist String ext = null; - String name = treeWalk.getPathString().toLowerCase(); + String name = blobPath.toLowerCase(); if (name.indexOf('.') > -1) { ext = name.substring(name.lastIndexOf('.') + 1); } @@ -298,25 +366,25 @@ public class LuceneUtils { treeWalk.release(); // index the head commit object - String head = rev.getId().getName(); if (indexedCommits.add(head)) { - Document doc = createDocument(rev, tags.get(head)); - doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, - Index.NOT_ANALYZED)); - doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED)); + Document doc = createDocument(branchHead, tags.get(head)); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED)); writer.addDocument(doc); result.commitCount += 1; + result.branchCount += 1; } // traverse the log and index the previous commit objects - revWalk.markStart(rev); + revWalk.reset(); + revWalk.markStart(branchHead); + RevCommit rev; while ((rev = revWalk.next()) != null) { String hash = rev.getId().getName(); if (indexedCommits.add(hash)) { Document doc = createDocument(rev, tags.get(hash)); - doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, - Index.NOT_ANALYZED)); - doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED)); writer.addDocument(doc); result.commitCount += 1; } @@ -329,10 +397,13 @@ public class LuceneUtils { // this repository has a gb-issues branch, index all issues if (IssueUtils.getIssuesBranch(repository) != null) { List issues = IssueUtils.getIssues(repository, null); + if (issues.size() > 0) { + result.branchCount += 1; + } for (IssueModel issue : issues) { + result.issueCount++; Document doc = createDocument(issue); - doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, - Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED)); writer.addDocument(doc); } } @@ -353,13 +424,16 @@ public class LuceneUtils { * Incrementally update the index with the specified commit for the * repository. * + * @param repositoryName * @param repository * @param branch * the fully qualified branch name (e.g. refs/heads/master) * @param commit * @return true, if successful */ - public static boolean index(Repository repository, String branch, RevCommit commit) { + private static IndexResult index(String repositoryName, Repository repository, + String branch, RevCommit commit) { + IndexResult result = new IndexResult(); try { if (excludedBranches.contains(branch)) { if (IssueUtils.GB_ISSUES.equals(branch)) { @@ -367,20 +441,23 @@ public class LuceneUtils { String issueId = commit.getShortMessage().substring(2).trim(); IssueModel issue = IssueUtils.getIssue(repository, issueId); if (issue == null) { - // delete the old issue from the index, if exists + // issue was deleted, remove from index IndexWriter writer = getIndexWriter(repository, false); writer.deleteDocuments( new Term(FIELD_OBJECT_TYPE, ObjectType.issue.name()), new Term( FIELD_OBJECT_ID, issueId)); writer.commit(); - return true; + result.success = true; + return result; } - return index(repository, issue); + result.success = index(repositoryName, repository, issue); + result.issueCount++; + return result; + } - return false; + return result; } List changedPaths = JGitUtils.getFilesInCommit(repository, commit); - String repositoryName = getName(repository); String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L, Resolution.MINUTE); IndexWriter writer = getIndexWriter(repository, false); @@ -391,19 +468,16 @@ public class LuceneUtils { // re-index the blob if (!ChangeType.DELETE.equals(path.changeType)) { + result.blobCount++; Document doc = new Document(); doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES, - Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.NOT_ANALYZED)); - doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.NOT_ANALYZED)); - doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES, Index.ANALYZED)); doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO)); - doc.add(new Field(FIELD_AUTHOR, commit.getAuthorIdent().getName(), Store.YES, - Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field(FIELD_COMMITTER, commit.getCommitterIdent().getName(), - Store.YES, Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field(FIELD_LABEL, branch, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED)); // determine extension to compare to the extension // blacklist @@ -425,11 +499,12 @@ public class LuceneUtils { writer.commit(); Document doc = createDocument(commit, null); - return index(repository, doc); + result.commitCount++; + result.success = index(repositoryName, repository, doc); } catch (Exception e) { e.printStackTrace(); } - return false; + return result; } /** @@ -440,7 +515,7 @@ public class LuceneUtils { * @param issue * @return true, if successful */ - public static boolean index(Repository repository, IssueModel issue) { + public static boolean index(String repositoryName, Repository repository, IssueModel issue) { try { // delete the old issue from the index, if exists IndexWriter writer = getIndexWriter(repository, false); @@ -449,7 +524,7 @@ public class LuceneUtils { writer.commit(); Document doc = createDocument(issue); - return index(repository, doc); + return index(repositoryName, repository, doc); } catch (Exception e) { e.printStackTrace(); } @@ -459,10 +534,11 @@ public class LuceneUtils { /** * Updates a repository index incrementally from the last indexed commits. * + * @param repositoryName * @param repository * @return IndexResult */ - public static IndexResult updateIndex(Repository repository) { + public static IndexResult updateIndex(String repositoryName, Repository repository) { IndexResult result = new IndexResult(); try { FileBasedConfig config = getConfig(repository); @@ -511,11 +587,14 @@ public class LuceneUtils { revs = JGitUtils.getRevLog(repository, lastCommit, branchName); } - // reverse the list of commits so we start with the first commit + if (revs.size() > 0) { + result.branchCount += 1; + } + + // reverse the list of commits so we start with the first commit Collections.reverse(revs); for (RevCommit commit : revs) { - index(repository, branchName, commit); - result.commitCount += 1; + result.add(index(repositoryName, repository, branchName, commit)); } // update the config @@ -550,12 +629,12 @@ public class LuceneUtils { private static Document createDocument(IssueModel issue) { Document doc = new Document(); doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.issue.name(), Store.YES, - Field.Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.NOT_ANALYZED)); - doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.NOT_ANALYZED)); + Field.Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.ANALYZED)); doc.add(new Field(FIELD_DATE, DateTools.dateToString(issue.created, Resolution.MINUTE), Store.YES, Field.Index.NO)); - doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.ANALYZED)); List attachments = new ArrayList(); for (Attachment attachment : issue.getAttachments()) { attachments.add(attachment.name.toLowerCase()); @@ -579,19 +658,16 @@ public class LuceneUtils { private static Document createDocument(RevCommit commit, List tags) { Document doc = new Document(); doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.commit.name(), Store.YES, - Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field(FIELD_OBJECT_ID, commit.getName(), Store.YES, Index.NOT_ANALYZED)); + Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_OBJECT_ID, commit.getName(), Store.YES, Index.ANALYZED)); doc.add(new Field(FIELD_DATE, DateTools.timeToString(commit.getCommitTime() * 1000L, Resolution.MINUTE), Store.YES, Index.NO)); - doc.add(new Field(FIELD_AUTHOR, commit.getCommitterIdent().getName(), Store.YES, - Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED)); doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), Store.YES, Index.ANALYZED)); doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), Store.NO, Index.ANALYZED)); if (!ArrayUtils.isEmpty(tags)) { - if (!ArrayUtils.isEmpty(tags)) { - doc.add(new Field(FIELD_LABEL, StringUtils.flattenStrings(tags), Store.YES, - Index.ANALYZED)); - } + doc.add(new Field(FIELD_TAG, StringUtils.flattenStrings(tags), Store.YES, Index.ANALYZED)); } return doc; } @@ -599,13 +675,13 @@ public class LuceneUtils { /** * Incrementally index an object for the repository. * + * @param repositoryName * @param repository * @param doc * @return true, if successful */ - private static boolean index(Repository repository, Document doc) { - try { - String repositoryName = getName(repository); + private static boolean index(String repositoryName, Repository repository, Document doc) { + try { doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.NOT_ANALYZED)); IndexWriter writer = getIndexWriter(repository, false); writer.addDocument(doc); @@ -629,6 +705,9 @@ public class LuceneUtils { result.repository = doc.get(FIELD_REPOSITORY); result.branch = doc.get(FIELD_BRANCH); result.id = doc.get(FIELD_OBJECT_ID); + if (doc.get(FIELD_TAG) != null) { + result.tags = StringUtils.getStringsFromValue(doc.get(FIELD_TAG)); + } if (doc.get(FIELD_LABEL) != null) { result.labels = StringUtils.getStringsFromValue(doc.get(FIELD_LABEL)); } @@ -672,7 +751,7 @@ public class LuceneUtils { private static IndexWriter getIndexWriter(Repository repository, boolean forceCreate) throws IOException { IndexWriter indexWriter = WRITERS.get(repository.getDirectory()); - File indexFolder = new File(repository.getDirectory(), "lucene"); + File indexFolder = new File(repository.getDirectory(), LUCENE_DIR); Directory directory = FSDirectory.open(indexFolder); if (forceCreate || !indexFolder.exists()) { // if the writer is going to blow away the existing index and create @@ -794,6 +873,16 @@ public class LuceneUtils { public static class IndexResult { public boolean success; + public int branchCount; public int commitCount; + public int blobCount; + public int issueCount; + + public void add(IndexResult result) { + this.branchCount += result.branchCount; + this.commitCount += result.commitCount; + this.blobCount += result.blobCount; + this.issueCount += result.issueCount; + } } } diff --git a/src/com/gitblit/wicket/GitBlitWebApp.java b/src/com/gitblit/wicket/GitBlitWebApp.java index e2391d6d..5054afa3 100644 --- a/src/com/gitblit/wicket/GitBlitWebApp.java +++ b/src/com/gitblit/wicket/GitBlitWebApp.java @@ -26,6 +26,7 @@ import org.apache.wicket.protocol.http.WebApplication; import com.gitblit.GitBlit; import com.gitblit.Keys; import com.gitblit.wicket.pages.ActivityPage; +import com.gitblit.wicket.pages.BasePage; import com.gitblit.wicket.pages.BlamePage; import com.gitblit.wicket.pages.BlobDiffPage; import com.gitblit.wicket.pages.BlobPage; @@ -90,7 +91,12 @@ public class GitBlitWebApp extends WebApplication { mount("/commitdiff", CommitDiffPage.class, "r", "h"); mount("/patch", PatchPage.class, "r", "h", "f"); mount("/history", HistoryPage.class, "r", "h", "f"); - mount("/search", SearchPage.class); + if (GitBlit.getBoolean(Keys.lucene.enable, false)) { + // TODO switch this to LucenePage when it is ready + mount("/search", SearchPage.class); + } else { + mount("/search", SearchPage.class); + } mount("/metrics", MetricsPage.class, "r"); mount("/blame", BlamePage.class, "r", "h", "f"); @@ -124,6 +130,20 @@ public class GitBlitWebApp extends WebApplication { public Class getHomePage() { return RepositoriesPage.class; } + + /** + * Returns the preferred search page class. + * + * @return a Wicket class representing a search page + */ + public Class getSearchPageClass() { + if (GitBlit.getBoolean(Keys.lucene.enable, false)) { + // TODO switch this to LucenePage when it is ready + return SearchPage.class;//LucenePage.class; + } + return SearchPage.class; + } + @Override public final Session newSession(Request request, Response response) { diff --git a/src/com/gitblit/wicket/GitBlitWebApp.properties b/src/com/gitblit/wicket/GitBlitWebApp.properties index b4501c33..9f4b3c62 100644 --- a/src/com/gitblit/wicket/GitBlitWebApp.properties +++ b/src/com/gitblit/wicket/GitBlitWebApp.properties @@ -212,4 +212,6 @@ gb.hookScriptsDescription = run Groovy scripts on pushes to this Gitblit server gb.reset = reset gb.pages = pages gb.workingCopy = working copy -gb.workingCopyWarning = this repository has a working copy and can not receive pushes \ No newline at end of file +gb.workingCopyWarning = this repository has a working copy and can not receive pushes +gb.query = query +gb.queryHelp = write brief explanation of Lucene syntax here \ No newline at end of file diff --git a/src/com/gitblit/wicket/pages/LucenePage.html b/src/com/gitblit/wicket/pages/LucenePage.html index 867e607b..ff911168 100644 --- a/src/com/gitblit/wicket/pages/LucenePage.html +++ b/src/com/gitblit/wicket/pages/LucenePage.html @@ -6,17 +6,46 @@
-

lucene

+

-
- - + +
+
+

+ +
+
+
+

+ + +
+
+
+
+ type: commit or blob
+ id: commit id or file path
+ branch:
+ author:
+ committer:
+ tag: tag
+
+
+
+ +
+
+
+
+
+
committed to :
+
\ No newline at end of file diff --git a/src/com/gitblit/wicket/pages/LucenePage.java b/src/com/gitblit/wicket/pages/LucenePage.java index 47f731c2..c269c8e6 100644 --- a/src/com/gitblit/wicket/pages/LucenePage.java +++ b/src/com/gitblit/wicket/pages/LucenePage.java @@ -20,7 +20,8 @@ import java.util.List; import org.apache.wicket.PageParameters; import org.apache.wicket.markup.html.basic.Label; -import org.apache.wicket.markup.html.form.Form; +import org.apache.wicket.markup.html.form.ListMultipleChoice; +import org.apache.wicket.markup.html.form.StatelessForm; import org.apache.wicket.markup.html.form.TextField; import org.apache.wicket.markup.repeater.Item; import org.apache.wicket.markup.repeater.data.DataView; @@ -29,6 +30,7 @@ import org.apache.wicket.model.Model; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; +import com.gitblit.Constants.SearchType; import com.gitblit.GitBlit; import com.gitblit.models.SearchResult; import com.gitblit.utils.LuceneUtils; @@ -50,6 +52,18 @@ public class LucenePage extends RootPage { private void setup(PageParameters params) { setupPage("", ""); + + String repository = null; + String value = null; + com.gitblit.Constants.SearchType searchType = null; + + if (params != null) { + repository = WicketUtils.getRepositoryName(params); + value = WicketUtils.getSearchString(params); + String type = WicketUtils.getSearchType(params); + searchType = com.gitblit.Constants.SearchType.forName(type); + } + final List results = new ArrayList(); ListDataProvider resultsDp = new ListDataProvider(results); final DataView resultsView = new DataView("searchResults", resultsDp) { @@ -86,26 +100,70 @@ public class LucenePage extends RootPage { } }; + // initial query final Model fragment = new Model(); - Form form = new Form("searchForm") { + if (!StringUtils.isEmpty(value)) { + if (searchType == SearchType.COMMIT) { + fragment.setObject("type:" + searchType.name().toLowerCase() + " AND \"" + value + "\""); + } else { + fragment.setObject(searchType.name().toLowerCase() + ":\"" + value + "\""); + } + } + + // selected repositories + final Model> repositories = new Model>(); + if (!StringUtils.isEmpty(repository)) { + ArrayList list = new ArrayList(); + list.add(repository); + repositories.setObject(list); + } + + // search form + StatelessForm form = new StatelessForm("searchForm") { + + private static final long serialVersionUID = 1L; + @Override public void onSubmit() { String f = fragment.getObject(); - List repositories = new ArrayList(); - for (String r : GitBlit.self().getRepositoryList()) { - repositories.add(GitBlit.self().getRepository(r)); + if (StringUtils.isEmpty(f)) { + error("Query is empty!"); + return; + } + if (repositories.getObject().size() == 0) { + error("Please select one or more repositories!"); + return; } - List srs = LuceneUtils.search(f, 100, repositories.toArray(new Repository[repositories.size()])); results.clear(); - results.addAll(srs); - for (Repository r : repositories) { - r.close(); - } + results.addAll(search(repositories, fragment)); resultsView.setVisible(true); } }; + ListMultipleChoice selections = new ListMultipleChoice("repositories", repositories, GitBlit.self().getRepositoryList()); + selections.setMaxRows(11); + form.add(selections); form.add(new TextField("fragment", fragment)); add(form); - add(resultsView.setVisible(false)); + if (!StringUtils.isEmpty(repository) && !StringUtils.isEmpty(fragment.getObject())) { + // search is defined by url parameters + results.clear(); + results.addAll(search(repositories, fragment)); + add(resultsView.setVisible(true)); + } else { + // no pre-defined search + add(resultsView.setVisible(false)); + } + } + + private List search(Model> repositories, Model fragment) { + List repos = new ArrayList(); + for (String r : repositories.getObject()) { + repos.add(GitBlit.self().getRepository(r)); + } + List srs = LuceneUtils.search(fragment.getObject(), 100, repos.toArray(new Repository[repos.size()])); + for (Repository r : repos) { + r.close(); + } + return srs; } } diff --git a/src/com/gitblit/wicket/pages/RepositoryPage.java b/src/com/gitblit/wicket/pages/RepositoryPage.java index 080a7723..4a83b876 100644 --- a/src/com/gitblit/wicket/pages/RepositoryPage.java +++ b/src/com/gitblit/wicket/pages/RepositoryPage.java @@ -47,6 +47,7 @@ import com.gitblit.models.RepositoryModel; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.StringUtils; import com.gitblit.utils.TicgitUtils; +import com.gitblit.wicket.GitBlitWebApp; import com.gitblit.wicket.GitBlitWebSession; import com.gitblit.wicket.PageRegistration; import com.gitblit.wicket.PageRegistration.OtherPageLink; @@ -228,6 +229,8 @@ public abstract class RepositoryPage extends BasePage { protected Component createPersonPanel(String wicketId, PersonIdent identity, Constants.SearchType searchType) { + GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication(); + final Class searchPageClass = app.getSearchPageClass(); String name = identity == null ? "" : identity.getName(); String address = identity == null ? "" : identity.getEmailAddress(); boolean showEmail = GitBlit.getBoolean(Keys.web.showEmailAddresses, false); @@ -241,20 +244,20 @@ public abstract class RepositoryPage extends BasePage { } } Fragment partial = new Fragment(wicketId, "partialPersonIdent", this); - LinkPanel link = new LinkPanel("personName", "list", value, SearchPage.class, + LinkPanel link = new LinkPanel("personName", "list", value, searchPageClass, WicketUtils.newSearchParameter(repositoryName, objectId, value, searchType)); setPersonSearchTooltip(link, value, searchType); partial.add(link); return partial; } else { Fragment fullPerson = new Fragment(wicketId, "fullPersonIdent", this); - LinkPanel nameLink = new LinkPanel("personName", "list", name, SearchPage.class, + LinkPanel nameLink = new LinkPanel("personName", "list", name, searchPageClass, WicketUtils.newSearchParameter(repositoryName, objectId, name, searchType)); setPersonSearchTooltip(nameLink, name, searchType); fullPerson.add(nameLink); LinkPanel addressLink = new LinkPanel("personAddress", "list", "<" + address + ">", - SearchPage.class, WicketUtils.newSearchParameter(repositoryName, objectId, + searchPageClass, WicketUtils.newSearchParameter(repositoryName, objectId, address, searchType)); setPersonSearchTooltip(addressLink, address, searchType); fullPerson.add(addressLink); @@ -357,7 +360,8 @@ public abstract class RepositoryPage extends BasePage { break; } } - setResponsePage(SearchPage.class, + GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication(); + setResponsePage(app.getSearchPageClass(), WicketUtils.newSearchParameter(repositoryName, null, searchString, searchType)); } } diff --git a/src/com/gitblit/wicket/panels/ActivityPanel.java b/src/com/gitblit/wicket/panels/ActivityPanel.java index dc94f3c1..37ab9f0c 100644 --- a/src/com/gitblit/wicket/panels/ActivityPanel.java +++ b/src/com/gitblit/wicket/panels/ActivityPanel.java @@ -28,7 +28,10 @@ import com.gitblit.Constants; import com.gitblit.models.Activity; import com.gitblit.models.Activity.RepositoryCommit; import com.gitblit.utils.StringUtils; +import com.gitblit.wicket.GitBlitWebApp; +import com.gitblit.wicket.GitBlitWebSession; import com.gitblit.wicket.WicketUtils; +import com.gitblit.wicket.pages.BasePage; import com.gitblit.wicket.pages.CommitDiffPage; import com.gitblit.wicket.pages.CommitPage; import com.gitblit.wicket.pages.LogPage; @@ -50,6 +53,9 @@ public class ActivityPanel extends BasePanel { super(wicketId); Collections.sort(recentActivity); + + GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication(); + final Class searchPageClass = app.getSearchPageClass(); DataView activityView = new DataView("activity", new ListDataProvider(recentActivity)) { @@ -86,7 +92,7 @@ public class ActivityPanel extends BasePanel { // author search link String author = commit.getAuthorIdent().getName(); LinkPanel authorLink = new LinkPanel("author", "list", author, - SearchPage.class, WicketUtils.newSearchParameter(commit.repository, + searchPageClass, WicketUtils.newSearchParameter(commit.repository, commit.getName(), author, Constants.SearchType.AUTHOR), true); setPersonSearchTooltip(authorLink, author, Constants.SearchType.AUTHOR); fragment.add(authorLink); diff --git a/src/com/gitblit/wicket/panels/BranchesPanel.java b/src/com/gitblit/wicket/panels/BranchesPanel.java index 0d2fe4a9..58862095 100644 --- a/src/com/gitblit/wicket/panels/BranchesPanel.java +++ b/src/com/gitblit/wicket/panels/BranchesPanel.java @@ -35,7 +35,10 @@ import com.gitblit.models.RefModel; import com.gitblit.models.RepositoryModel; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.StringUtils; +import com.gitblit.wicket.GitBlitWebApp; +import com.gitblit.wicket.GitBlitWebSession; import com.gitblit.wicket.WicketUtils; +import com.gitblit.wicket.pages.BasePage; import com.gitblit.wicket.pages.BranchesPage; import com.gitblit.wicket.pages.CommitPage; import com.gitblit.wicket.pages.LogPage; @@ -75,6 +78,9 @@ public class BranchesPanel extends BasePanel { add(new Label("branches", new StringResourceModel("gb.branches", this, null))); } + GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication(); + final Class searchPageClass = app.getSearchPageClass(); + ListDataProvider branchesDp = new ListDataProvider(branches); DataView branchesView = new DataView("branch", branchesDp) { private static final long serialVersionUID = 1L; @@ -91,7 +97,7 @@ public class BranchesPanel extends BasePanel { String author = entry.getAuthorIdent().getName(); LinkPanel authorLink = new LinkPanel("branchAuthor", "list", author, - SearchPage.class, WicketUtils.newSearchParameter(model.name, + searchPageClass, WicketUtils.newSearchParameter(model.name, entry.getName(), author, Constants.SearchType.AUTHOR)); setPersonSearchTooltip(authorLink, author, Constants.SearchType.AUTHOR); item.add(authorLink); diff --git a/src/com/gitblit/wicket/panels/HistoryPanel.java b/src/com/gitblit/wicket/panels/HistoryPanel.java index 36a76806..23eb2b3f 100644 --- a/src/com/gitblit/wicket/panels/HistoryPanel.java +++ b/src/com/gitblit/wicket/panels/HistoryPanel.java @@ -38,7 +38,10 @@ import com.gitblit.models.PathModel.PathChangeModel; import com.gitblit.models.RefModel; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.StringUtils; +import com.gitblit.wicket.GitBlitWebApp; +import com.gitblit.wicket.GitBlitWebSession; import com.gitblit.wicket.WicketUtils; +import com.gitblit.wicket.pages.BasePage; import com.gitblit.wicket.pages.BlobDiffPage; import com.gitblit.wicket.pages.BlobPage; import com.gitblit.wicket.pages.CommitDiffPage; @@ -94,6 +97,9 @@ public class HistoryPanel extends BasePanel { // breadcrumbs add(new PathBreadcrumbsPanel("breadcrumbs", repositoryName, path, objectId)); + GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication(); + final Class searchPageClass = app.getSearchPageClass(); + ListDataProvider dp = new ListDataProvider(commits); DataView logView = new DataView("commit", dp) { private static final long serialVersionUID = 1L; @@ -108,7 +114,7 @@ public class HistoryPanel extends BasePanel { // author search link String author = entry.getAuthorIdent().getName(); LinkPanel authorLink = new LinkPanel("commitAuthor", "list", author, - SearchPage.class, WicketUtils.newSearchParameter(repositoryName, objectId, + searchPageClass, WicketUtils.newSearchParameter(repositoryName, objectId, author, Constants.SearchType.AUTHOR)); setPersonSearchTooltip(authorLink, author, Constants.SearchType.AUTHOR); item.add(authorLink); diff --git a/src/com/gitblit/wicket/panels/LogPanel.java b/src/com/gitblit/wicket/panels/LogPanel.java index a960f6e0..380cc315 100644 --- a/src/com/gitblit/wicket/panels/LogPanel.java +++ b/src/com/gitblit/wicket/panels/LogPanel.java @@ -35,7 +35,10 @@ import com.gitblit.Keys; import com.gitblit.models.RefModel; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.StringUtils; +import com.gitblit.wicket.GitBlitWebApp; +import com.gitblit.wicket.GitBlitWebSession; import com.gitblit.wicket.WicketUtils; +import com.gitblit.wicket.pages.BasePage; import com.gitblit.wicket.pages.CommitDiffPage; import com.gitblit.wicket.pages.CommitPage; import com.gitblit.wicket.pages.LogPage; @@ -82,6 +85,9 @@ public class LogPanel extends BasePanel { WicketUtils.newRepositoryParameter(repositoryName))); } + GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication(); + final Class searchPageClass = app.getSearchPageClass(); + ListDataProvider dp = new ListDataProvider(commits); DataView logView = new DataView("commit", dp) { private static final long serialVersionUID = 1L; @@ -96,7 +102,7 @@ public class LogPanel extends BasePanel { // author search link String author = entry.getAuthorIdent().getName(); LinkPanel authorLink = new LinkPanel("commitAuthor", "list", author, - SearchPage.class, WicketUtils.newSearchParameter(repositoryName, objectId, + searchPageClass, WicketUtils.newSearchParameter(repositoryName, objectId, author, Constants.SearchType.AUTHOR)); setPersonSearchTooltip(authorLink, author, Constants.SearchType.AUTHOR); item.add(authorLink); diff --git a/tests/com/gitblit/tests/IssuesTest.java b/tests/com/gitblit/tests/IssuesTest.java index 92952119..eb7b66dd 100644 --- a/tests/com/gitblit/tests/IssuesTest.java +++ b/tests/com/gitblit/tests/IssuesTest.java @@ -36,6 +36,7 @@ import com.gitblit.models.SearchResult; import com.gitblit.utils.IssueUtils; import com.gitblit.utils.IssueUtils.IssueFilter; import com.gitblit.utils.LuceneUtils; +import com.gitblit.utils.StringUtils; /** * Tests the mechanics of distributed issue management on the gb-issues branch. @@ -48,6 +49,8 @@ public class IssuesTest { @Test public void testLifecycle() throws Exception { Repository repository = GitBlitSuite.getIssuesTestRepository(); + String name = StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(), + repository.getDirectory().getAbsolutePath()); // create and insert an issue Change c1 = newChange("testCreation() " + Long.toHexString(System.currentTimeMillis())); @@ -128,7 +131,7 @@ public class IssuesTest { // build a new Lucene index LuceneUtils.deleteIndex(repository); for (IssueModel anIssue : allIssues) { - LuceneUtils.index(repository, anIssue); + LuceneUtils.index(name, repository, anIssue); } List hits = LuceneUtils.search("working", 10, repository); assertTrue(hits.size() > 0); @@ -139,7 +142,7 @@ public class IssuesTest { change.comment("this is a test of reindexing an issue"); IssueUtils.updateIssue(repository, issue.id, change); issue = IssueUtils.getIssue(repository, issue.id); - LuceneUtils.index(repository, issue); + LuceneUtils.index(name, repository, issue); // delete all issues for (IssueModel anIssue : allIssues) { diff --git a/tests/com/gitblit/tests/LuceneUtilsTest.java b/tests/com/gitblit/tests/LuceneUtilsTest.java index 5e388237..53710090 100644 --- a/tests/com/gitblit/tests/LuceneUtilsTest.java +++ b/tests/com/gitblit/tests/LuceneUtilsTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import com.gitblit.models.SearchResult; import com.gitblit.utils.LuceneUtils; +import com.gitblit.utils.StringUtils; /** * Tests Lucene indexing and querying. @@ -37,19 +38,25 @@ public class LuceneUtilsTest { public void testFullIndex() throws Exception { // reindex helloworld Repository repository = GitBlitSuite.getHelloworldRepository(); - LuceneUtils.reindex(repository); + String name = StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(), + repository.getDirectory().getAbsolutePath()); + LuceneUtils.reindex(name, repository, false); repository.close(); // reindex theoretical physics repository = GitBlitSuite.getTheoreticalPhysicsRepository(); - LuceneUtils.reindex(repository); + name = StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(), + repository.getDirectory().getAbsolutePath()); + LuceneUtils.reindex(name, repository, false); repository.close(); - + // reindex JGit repository = GitBlitSuite.getJGitRepository(); - LuceneUtils.reindex(repository); + name = StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(), + repository.getDirectory().getAbsolutePath()); + LuceneUtils.reindex(name, repository, false); repository.close(); - + LuceneUtils.close(); } -- 2.39.5