+/*\r
+ * Copyright 2012 gitblit.com.\r
+ *\r
+ * Licensed under the Apache License, Version 2.0 (the "License");\r
+ * you may not use this file except in compliance with the License.\r
+ * You may obtain a copy of the License at\r
+ *\r
+ * http://www.apache.org/licenses/LICENSE-2.0\r
+ *\r
+ * Unless required by applicable law or agreed to in writing, software\r
+ * distributed under the License is distributed on an "AS IS" BASIS,\r
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r
+ * See the License for the specific language governing permissions and\r
+ * limitations under the License.\r
+ */\r
package com.gitblit.utils;\r
\r
import java.io.ByteArrayOutputStream;\r
import org.eclipse.jgit.revwalk.RevWalk;\r
import org.eclipse.jgit.storage.file.FileBasedConfig;\r
import org.eclipse.jgit.treewalk.TreeWalk;\r
+import org.eclipse.jgit.treewalk.filter.AndTreeFilter;\r
+import org.eclipse.jgit.treewalk.filter.PathFilterGroup;\r
+import org.eclipse.jgit.treewalk.filter.TreeFilter;\r
import org.eclipse.jgit.util.FS;\r
\r
-import com.gitblit.GitBlit;\r
import com.gitblit.models.IssueModel;\r
import com.gitblit.models.IssueModel.Attachment;\r
import com.gitblit.models.PathModel.PathChangeModel;\r
private static final String FIELD_AUTHOR = "author";\r
private static final String FIELD_COMMITTER = "committer";\r
private static final String FIELD_DATE = "date";\r
+ private static final String FIELD_TAG = "tag";\r
private static final String FIELD_LABEL = "label";\r
private static final String FIELD_ATTACHMENT = "attachment";\r
\r
private static final Map<File, IndexSearcher> SEARCHERS = new ConcurrentHashMap<File, IndexSearcher>();\r
private static final Map<File, IndexWriter> WRITERS = new ConcurrentHashMap<File, IndexWriter>();\r
\r
+ private static final String LUCENE_DIR = "lucene";\r
private static final String CONF_FILE = "lucene.conf";\r
private static final String CONF_INDEX = "index";\r
private static final String CONF_VERSION = "version";\r
private static final String CONF_ALIAS = "aliases";\r
private static final String CONF_BRANCH = "branches";\r
-\r
+ \r
/**\r
- * Returns the name of the repository.\r
+ * Returns the author for the commit, if this information is available.\r
* \r
- * @param repository\r
- * @return the repository name\r
+ * @param commit\r
+ * @return an author or unknown\r
*/\r
- private static String getName(Repository repository) {\r
- String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath();\r
- if (repository.isBare()) {\r
- return StringUtils.getRelativePath(rootPath, repository.getDirectory()\r
- .getAbsolutePath());\r
- } else {\r
- return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile()\r
- .getAbsolutePath());\r
+ private static String getAuthor(RevCommit commit) {\r
+ String name = "unknown";\r
+ try {\r
+ name = commit.getAuthorIdent().getName();\r
+ if (StringUtils.isEmpty(name)) {\r
+ name = commit.getAuthorIdent().getEmailAddress();\r
+ }\r
+ } catch (NullPointerException n) { \r
}\r
+ return name;\r
+ }\r
+ \r
+ /**\r
+ * Returns the committer for the commit, if this information is available.\r
+ * \r
+ * @param commit\r
+ * @return an committer or unknown\r
+ */\r
+ private static String getCommitter(RevCommit commit) {\r
+ String name = "unknown";\r
+ try {\r
+ name = commit.getCommitterIdent().getName();\r
+ if (StringUtils.isEmpty(name)) {\r
+ name = commit.getCommitterIdent().getEmailAddress();\r
+ }\r
+ } catch (NullPointerException n) { \r
+ }\r
+ return name;\r
}\r
\r
/**\r
*/\r
public static boolean deleteIndex(Repository repository) {\r
try {\r
- File luceneIndex = new File(repository.getDirectory(), "lucene");\r
+ File luceneIndex = new File(repository.getDirectory(), LUCENE_DIR);\r
if (luceneIndex.exists()) {\r
org.eclipse.jgit.util.FileUtils.delete(luceneIndex,\r
org.eclipse.jgit.util.FileUtils.RECURSIVE);\r
* This completely indexes the repository and will destroy any existing\r
* index.\r
* \r
+ * @param repositoryName\r
* @param repository\r
+ * @param fullIndex\r
+ * If false blob metadata is set to the HEAD revision of each\r
+ * branch. If true, each the last commit of each blob is\r
+ * determined to properly index the author, committer, and date.\r
+ * Full indexing can be time-consuming.\r
* @return IndexResult\r
*/\r
- public static IndexResult reindex(Repository repository) {\r
+ public static IndexResult reindex(String repositoryName, Repository repository,\r
+ boolean fullIndex) {\r
IndexResult result = new IndexResult();\r
if (!LuceneUtils.deleteIndex(repository)) {\r
return result;\r
}\r
- try {\r
- String repositoryName = getName(repository);\r
+ try { \r
FileBasedConfig config = getConfig(repository);\r
Set<String> indexedCommits = new TreeSet<String>();\r
IndexWriter writer = getIndexWriter(repository, true);\r
}\r
String branchName = branch.getName();\r
RevWalk revWalk = new RevWalk(repository);\r
- RevCommit rev = revWalk.parseCommit(branch.getObjectId());\r
+ RevCommit branchHead = revWalk.parseCommit(branch.getObjectId());\r
+ String head = branchHead.getId().getName();\r
\r
String keyName = getBranchKey(branchName);\r
config.setString(CONF_ALIAS, null, keyName, branchName);\r
- config.setString(CONF_BRANCH, null, keyName, rev.getName());\r
+ config.setString(CONF_BRANCH, null, keyName, head);\r
\r
// index the blob contents of the tree\r
ByteArrayOutputStream os = new ByteArrayOutputStream();\r
byte[] tmp = new byte[32767];\r
TreeWalk treeWalk = new TreeWalk(repository);\r
- treeWalk.addTree(rev.getTree());\r
+ treeWalk.addTree(branchHead.getTree());\r
treeWalk.setRecursive(true);\r
- String revDate = DateTools.timeToString(rev.getCommitTime() * 1000L,\r
- Resolution.MINUTE);\r
+ \r
while (treeWalk.next()) {\r
+ result.blobCount++;\r
+ String blobPath = treeWalk.getPathString();\r
+ RevCommit blobRev = branchHead;\r
+ \r
+ RevWalk blobWalk = null;\r
+ if (fullIndex) {\r
+ // XXX this is _really_ slow, there must be a better way\r
+ // determine the most recent commit for this blob\r
+ blobWalk = new RevWalk(repository);\r
+ blobWalk.markStart(blobWalk.parseCommit(branch.getObjectId()));\r
+ TreeFilter filter = AndTreeFilter.create(\r
+ PathFilterGroup.createFromStrings(Collections.singleton(blobPath)),\r
+ TreeFilter.ANY_DIFF);\r
+ blobWalk.setTreeFilter(filter);\r
+\r
+ for (RevCommit commit : blobWalk) {\r
+ blobRev = commit;\r
+ break;\r
+ }\r
+ }\r
+ \r
+ String blobAuthor = getAuthor(blobRev);\r
+ String blobCommitter = getCommitter(blobRev);\r
+ String blobDate = DateTools.timeToString(blobRev.getCommitTime() * 1000L,\r
+ Resolution.MINUTE);\r
+ \r
+ if (blobWalk != null) {\r
+ blobWalk.dispose();\r
+ }\r
+ \r
Document doc = new Document();\r
- doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,\r
- Index.NOT_ANALYZED_NO_NORMS));\r
- doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,\r
- Index.NOT_ANALYZED));\r
- doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));\r
- doc.add(new Field(FIELD_OBJECT_ID, treeWalk.getPathString(), Store.YES,\r
- Index.NOT_ANALYZED));\r
- doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));\r
- doc.add(new Field(FIELD_AUTHOR, rev.getAuthorIdent().getName(), Store.YES,\r
- Index.NOT_ANALYZED_NO_NORMS));\r
- doc.add(new Field(FIELD_COMMITTER, rev.getCommitterIdent().getName(),\r
- Store.YES, Index.NOT_ANALYZED_NO_NORMS));\r
- doc.add(new Field(FIELD_LABEL, branch.getName(), Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES, Index.NOT_ANALYZED_NO_NORMS));\r
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_OBJECT_ID, blobPath, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_DATE, blobDate, Store.YES, Index.NO));\r
+ doc.add(new Field(FIELD_AUTHOR, blobAuthor, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_COMMITTER, blobCommitter, Store.YES, Index.ANALYZED)); \r
\r
// determine extension to compare to the extension\r
// blacklist\r
String ext = null;\r
- String name = treeWalk.getPathString().toLowerCase();\r
+ String name = blobPath.toLowerCase();\r
if (name.indexOf('.') > -1) {\r
ext = name.substring(name.lastIndexOf('.') + 1);\r
}\r
treeWalk.release();\r
\r
// index the head commit object\r
- String head = rev.getId().getName();\r
if (indexedCommits.add(head)) {\r
- Document doc = createDocument(rev, tags.get(head));\r
- doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,\r
- Index.NOT_ANALYZED));\r
- doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));\r
+ Document doc = createDocument(branchHead, tags.get(head));\r
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));\r
writer.addDocument(doc);\r
result.commitCount += 1;\r
+ result.branchCount += 1;\r
}\r
\r
// traverse the log and index the previous commit objects\r
- revWalk.markStart(rev);\r
+ revWalk.reset();\r
+ revWalk.markStart(branchHead);\r
+ RevCommit rev;\r
while ((rev = revWalk.next()) != null) {\r
String hash = rev.getId().getName();\r
if (indexedCommits.add(hash)) {\r
Document doc = createDocument(rev, tags.get(hash));\r
- doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,\r
- Index.NOT_ANALYZED));\r
- doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));\r
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));\r
writer.addDocument(doc);\r
result.commitCount += 1;\r
}\r
// this repository has a gb-issues branch, index all issues\r
if (IssueUtils.getIssuesBranch(repository) != null) {\r
List<IssueModel> issues = IssueUtils.getIssues(repository, null);\r
+ if (issues.size() > 0) {\r
+ result.branchCount += 1;\r
+ }\r
for (IssueModel issue : issues) {\r
+ result.issueCount++;\r
Document doc = createDocument(issue);\r
- doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,\r
- Index.NOT_ANALYZED));\r
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));\r
writer.addDocument(doc);\r
}\r
}\r
* Incrementally update the index with the specified commit for the\r
* repository.\r
* \r
+ * @param repositoryName\r
* @param repository\r
* @param branch\r
* the fully qualified branch name (e.g. refs/heads/master)\r
* @param commit\r
* @return true, if successful\r
*/\r
- public static boolean index(Repository repository, String branch, RevCommit commit) {\r
+ private static IndexResult index(String repositoryName, Repository repository, \r
+ String branch, RevCommit commit) {\r
+ IndexResult result = new IndexResult();\r
try {\r
if (excludedBranches.contains(branch)) {\r
if (IssueUtils.GB_ISSUES.equals(branch)) {\r
String issueId = commit.getShortMessage().substring(2).trim();\r
IssueModel issue = IssueUtils.getIssue(repository, issueId);\r
if (issue == null) {\r
- // delete the old issue from the index, if exists\r
+ // issue was deleted, remove from index\r
IndexWriter writer = getIndexWriter(repository, false);\r
writer.deleteDocuments(\r
new Term(FIELD_OBJECT_TYPE, ObjectType.issue.name()), new Term(\r
FIELD_OBJECT_ID, issueId));\r
writer.commit();\r
- return true;\r
+ result.success = true;\r
+ return result;\r
}\r
- return index(repository, issue);\r
+ result.success = index(repositoryName, repository, issue);\r
+ result.issueCount++;\r
+ return result;\r
+ \r
}\r
- return false;\r
+ return result;\r
}\r
List<PathChangeModel> changedPaths = JGitUtils.getFilesInCommit(repository, commit);\r
- String repositoryName = getName(repository);\r
String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L,\r
Resolution.MINUTE);\r
IndexWriter writer = getIndexWriter(repository, false);\r
\r
// re-index the blob\r
if (!ChangeType.DELETE.equals(path.changeType)) {\r
+ result.blobCount++;\r
Document doc = new Document();\r
doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,\r
- Index.NOT_ANALYZED_NO_NORMS));\r
- doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,\r
Index.NOT_ANALYZED));\r
- doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.NOT_ANALYZED));\r
- doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES, Index.NOT_ANALYZED));\r
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES, Index.ANALYZED));\r
doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));\r
- doc.add(new Field(FIELD_AUTHOR, commit.getAuthorIdent().getName(), Store.YES,\r
- Index.NOT_ANALYZED_NO_NORMS));\r
- doc.add(new Field(FIELD_COMMITTER, commit.getCommitterIdent().getName(),\r
- Store.YES, Index.NOT_ANALYZED_NO_NORMS));\r
- doc.add(new Field(FIELD_LABEL, branch, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));\r
\r
// determine extension to compare to the extension\r
// blacklist\r
writer.commit();\r
\r
Document doc = createDocument(commit, null);\r
- return index(repository, doc);\r
+ result.commitCount++;\r
+ result.success = index(repositoryName, repository, doc);\r
} catch (Exception e) {\r
e.printStackTrace();\r
}\r
- return false;\r
+ return result;\r
}\r
\r
/**\r
* @param issue\r
* @return true, if successful\r
*/\r
- public static boolean index(Repository repository, IssueModel issue) {\r
+ public static boolean index(String repositoryName, Repository repository, IssueModel issue) {\r
try {\r
// delete the old issue from the index, if exists\r
IndexWriter writer = getIndexWriter(repository, false);\r
writer.commit();\r
\r
Document doc = createDocument(issue);\r
- return index(repository, doc);\r
+ return index(repositoryName, repository, doc);\r
} catch (Exception e) {\r
e.printStackTrace();\r
}\r
/**\r
* Updates a repository index incrementally from the last indexed commits.\r
* \r
+ * @param repositoryName\r
* @param repository\r
* @return IndexResult\r
*/\r
- public static IndexResult updateIndex(Repository repository) {\r
+ public static IndexResult updateIndex(String repositoryName, Repository repository) {\r
IndexResult result = new IndexResult();\r
try {\r
FileBasedConfig config = getConfig(repository);\r
revs = JGitUtils.getRevLog(repository, lastCommit, branchName);\r
}\r
\r
- // reverse the list of commits so we start with the first commit\r
+ if (revs.size() > 0) {\r
+ result.branchCount += 1;\r
+ }\r
+ \r
+ // reverse the list of commits so we start with the first commit \r
Collections.reverse(revs);\r
for (RevCommit commit : revs) {\r
- index(repository, branchName, commit);\r
- result.commitCount += 1;\r
+ result.add(index(repositoryName, repository, branchName, commit)); \r
}\r
\r
// update the config\r
private static Document createDocument(IssueModel issue) {\r
Document doc = new Document();\r
doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.issue.name(), Store.YES,\r
- Field.Index.NOT_ANALYZED_NO_NORMS));\r
- doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.NOT_ANALYZED));\r
- doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.NOT_ANALYZED));\r
+ Field.Index.NOT_ANALYZED));\r
+ doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.ANALYZED));\r
doc.add(new Field(FIELD_DATE, DateTools.dateToString(issue.created, Resolution.MINUTE),\r
Store.YES, Field.Index.NO));\r
- doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.NOT_ANALYZED_NO_NORMS));\r
+ doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.ANALYZED));\r
List<String> attachments = new ArrayList<String>();\r
for (Attachment attachment : issue.getAttachments()) {\r
attachments.add(attachment.name.toLowerCase());\r
private static Document createDocument(RevCommit commit, List<String> tags) {\r
Document doc = new Document();\r
doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.commit.name(), Store.YES,\r
- Index.NOT_ANALYZED_NO_NORMS));\r
- doc.add(new Field(FIELD_OBJECT_ID, commit.getName(), Store.YES, Index.NOT_ANALYZED));\r
+ Index.NOT_ANALYZED));\r
+ doc.add(new Field(FIELD_OBJECT_ID, commit.getName(), Store.YES, Index.ANALYZED));\r
doc.add(new Field(FIELD_DATE, DateTools.timeToString(commit.getCommitTime() * 1000L,\r
Resolution.MINUTE), Store.YES, Index.NO));\r
- doc.add(new Field(FIELD_AUTHOR, commit.getCommitterIdent().getName(), Store.YES,\r
- Index.NOT_ANALYZED_NO_NORMS));\r
+ doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));\r
+ doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));\r
doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), Store.YES, Index.ANALYZED));\r
doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), Store.NO, Index.ANALYZED));\r
if (!ArrayUtils.isEmpty(tags)) {\r
- if (!ArrayUtils.isEmpty(tags)) {\r
- doc.add(new Field(FIELD_LABEL, StringUtils.flattenStrings(tags), Store.YES,\r
- Index.ANALYZED));\r
- }\r
+ doc.add(new Field(FIELD_TAG, StringUtils.flattenStrings(tags), Store.YES, Index.ANALYZED));\r
}\r
return doc;\r
}\r
/**\r
* Incrementally index an object for the repository.\r
* \r
+ * @param repositoryName\r
* @param repository\r
* @param doc\r
* @return true, if successful\r
*/\r
- private static boolean index(Repository repository, Document doc) {\r
- try {\r
- String repositoryName = getName(repository);\r
+ private static boolean index(String repositoryName, Repository repository, Document doc) {\r
+ try { \r
doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.NOT_ANALYZED));\r
IndexWriter writer = getIndexWriter(repository, false);\r
writer.addDocument(doc);\r
result.repository = doc.get(FIELD_REPOSITORY);\r
result.branch = doc.get(FIELD_BRANCH);\r
result.id = doc.get(FIELD_OBJECT_ID);\r
+ if (doc.get(FIELD_TAG) != null) {\r
+ result.tags = StringUtils.getStringsFromValue(doc.get(FIELD_TAG));\r
+ }\r
if (doc.get(FIELD_LABEL) != null) {\r
result.labels = StringUtils.getStringsFromValue(doc.get(FIELD_LABEL));\r
}\r
private static IndexWriter getIndexWriter(Repository repository, boolean forceCreate)\r
throws IOException {\r
IndexWriter indexWriter = WRITERS.get(repository.getDirectory());\r
- File indexFolder = new File(repository.getDirectory(), "lucene");\r
+ File indexFolder = new File(repository.getDirectory(), LUCENE_DIR);\r
Directory directory = FSDirectory.open(indexFolder);\r
if (forceCreate || !indexFolder.exists()) {\r
// if the writer is going to blow away the existing index and create\r
\r
public static class IndexResult {\r
public boolean success;\r
+ public int branchCount;\r
public int commitCount;\r
+ public int blobCount;\r
+ public int issueCount;\r
+ \r
+ public void add(IndexResult result) {\r
+ this.branchCount += result.branchCount;\r
+ this.commitCount += result.commitCount;\r
+ this.blobCount += result.blobCount;\r
+ this.issueCount += result.issueCount; \r
+ }\r
}\r
}\r