From 0f43a54527845b5873f35dc80300d578bfe84bb0 Mon Sep 17 00:00:00 2001 From: James Moger Date: Fri, 13 Jan 2012 07:58:12 -0500 Subject: Branch for implementing distributed gb-issues --- src/com/gitblit/models/IssueModel.java | 310 ++++++++++++++++++++++ src/com/gitblit/utils/IssueUtils.java | 455 +++++++++++++++++++++++++++++++++ src/com/gitblit/utils/JGitUtils.java | 33 ++- src/com/gitblit/utils/JsonUtils.java | 34 ++- 4 files changed, 821 insertions(+), 11 deletions(-) create mode 100644 src/com/gitblit/models/IssueModel.java create mode 100644 src/com/gitblit/utils/IssueUtils.java (limited to 'src/com') diff --git a/src/com/gitblit/models/IssueModel.java b/src/com/gitblit/models/IssueModel.java new file mode 100644 index 00000000..3c6d9a0b --- /dev/null +++ b/src/com/gitblit/models/IssueModel.java @@ -0,0 +1,310 @@ +/* + * Copyright 2012 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.models; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import com.gitblit.utils.ArrayUtils; +import com.gitblit.utils.StringUtils; + +/** + * The Gitblit Issue model, its component classes, and enums. + * + * @author James Moger + * + */ +public class IssueModel implements Serializable, Comparable { + + private static final long serialVersionUID = 1L;; + + public String id; + + public Type type; + + public Status status; + + public Priority priority; + + public Date created; + + public String summary; + + public String description; + + public String reporter; + + public String owner; + + public String milestone; + + public List changes; + + public IssueModel() { + created = new Date(); + + type = Type.Defect; + status = Status.New; + priority = Priority.Medium; + + changes = new ArrayList(); + } + + public String getStatus() { + String s = status.toString(); + if (!StringUtils.isEmpty(owner)) + s += " (" + owner + ")"; + return s; + } + + public List getLabels() { + List list = new ArrayList(); + String labels = null; + for (Change change : changes) { + if (change.hasFieldChanges()) { + FieldChange field = change.getField(Field.Labels); + if (field != null) { + labels = field.value.toString(); + } + } + } + if (!StringUtils.isEmpty(labels)) { + list.addAll(StringUtils.getStringsFromValue(labels, " ")); + } + return list; + } + + public boolean hasLabel(String label) { + return getLabels().contains(label); + } + + public Attachment getAttachment(String name) { + Attachment attachment = null; + for (Change change : changes) { + if (change.hasAttachments()) { + Attachment a = change.getAttachment(name); + if (a != null) { + attachment = a; + } + } + } + return attachment; + } + + public void addChange(Change change) { + if (changes == null) { + changes = new ArrayList(); + } + changes.add(change); + } + + @Override + public String toString() { + return summary; + } + + @Override + public int compareTo(IssueModel o) { + return o.created.compareTo(created); + } + + @Override + public boolean equals(Object o) { + if (o instanceof IssueModel) + return id.equals(((IssueModel) o).id); + return super.equals(o); + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + public static class Change implements Serializable { + + private static final long serialVersionUID = 1L; + + public Date created; + + public String author; + + public Comment comment; + + public List fieldChanges; + + public List attachments; + + public void comment(String text) { + comment = new Comment(text); + } + + public boolean hasComment() { + return comment != null; + } + + public boolean hasAttachments() { + return !ArrayUtils.isEmpty(attachments); + } + + public boolean hasFieldChanges() { + return !ArrayUtils.isEmpty(fieldChanges); + } + + public FieldChange getField(Field field) { + if (fieldChanges != null) { + for (FieldChange fieldChange : fieldChanges) { + if (fieldChange.field == field) { + return fieldChange; + } + } + } + return null; + } + + public void setField(Field field, Object value) { + FieldChange fieldChange = new FieldChange(); + fieldChange.field = field; + fieldChange.value = value; + if (fieldChanges == null) { + fieldChanges = new ArrayList(); + } + fieldChanges.add(fieldChange); + } + + public String getString(Field field) { + FieldChange fieldChange = getField(field); + if (fieldChange == null) { + return null; + } + return fieldChange.value.toString(); + } + + public void addAttachment(Attachment attachment) { + if (attachments == null) { + attachments = new ArrayList(); + } + attachments.add(attachment); + } + + public Attachment getAttachment(String name) { + for (Attachment attachment : attachments) { + if (attachment.name.equalsIgnoreCase(name)) { + return attachment; + } + } + return null; + } + + @Override + public String toString() { + return created.toString() + " by " + author; + } + } + + public static class Comment implements Serializable { + + private static final long serialVersionUID = 1L; + + public String text; + public boolean deleted; + + Comment(String text) { + this.text = text; + } + + @Override + public String toString() { + return text; + } + } + + public static class FieldChange implements Serializable { + + private static final long serialVersionUID = 1L; + + public Field field; + + public Object value; + + @Override + public String toString() { + return field + ": " + value; + } + } + + public static class Attachment implements Serializable { + + private static final long serialVersionUID = 1L; + + public String name; + public long size; + public byte[] content; + public boolean deleted; + + public Attachment(String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } + } + + public static enum Field { + Summary, Description, Reporter, Owner, Type, Status, Priority, Milestone, Labels; + } + + public static enum Type { + Defect, Enhancement, Task, Review, Other; + } + + public static enum Priority { + Low, Medium, High, Critical; + } + + public static enum Status { + New, Accepted, Started, Review, Queued, Testing, Done, Fixed, WontFix, Duplicate, Invalid; + + public boolean atLeast(Status status) { + return ordinal() >= status.ordinal(); + } + + public boolean exceeds(Status status) { + return ordinal() > status.ordinal(); + } + + public Status next() { + switch (this) { + case New: + return Started; + case Accepted: + return Started; + case Started: + return Testing; + case Review: + return Testing; + case Queued: + return Testing; + case Testing: + return Done; + } + return Accepted; + } + } +} diff --git a/src/com/gitblit/utils/IssueUtils.java b/src/com/gitblit/utils/IssueUtils.java new file mode 100644 index 00000000..82170703 --- /dev/null +++ b/src/com/gitblit/utils/IssueUtils.java @@ -0,0 +1,455 @@ +/* + * Copyright 2012 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.utils; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.eclipse.jgit.JGitText; +import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheBuilder; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.RefUpdate.Result; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.treewalk.TreeWalk; + +import com.gitblit.models.IssueModel; +import com.gitblit.models.IssueModel.Attachment; +import com.gitblit.models.IssueModel.Change; +import com.gitblit.models.IssueModel.Field; +import com.gitblit.models.PathModel; +import com.gitblit.models.RefModel; +import com.gitblit.utils.JsonUtils.ExcludeField; +import com.google.gson.Gson; + +/** + * Utility class for reading Gitblit issues. + * + * @author James Moger + * + */ +public class IssueUtils { + + public static final String GB_ISSUES = "refs/heads/gb-issues"; + + /** + * Returns a RefModel for the gb-issues branch in the repository. If the + * branch can not be found, null is returned. + * + * @param repository + * @return a refmodel for the gb-issues branch or null + */ + public static RefModel getIssuesBranch(Repository repository) { + return JGitUtils.getBranch(repository, "gb-issues"); + } + + /** + * Returns all the issues in the repository. + * + * @param repository + * @param filter + * optional issue filter to only return matching results + * @return a list of issues + */ + public static List getIssues(Repository repository, IssueFilter filter) { + List list = new ArrayList(); + RefModel issuesBranch = getIssuesBranch(repository); + if (issuesBranch == null) { + return list; + } + List paths = JGitUtils + .getDocuments(repository, Arrays.asList("json"), GB_ISSUES); + RevTree tree = JGitUtils.getCommit(repository, GB_ISSUES).getTree(); + for (PathModel path : paths) { + String json = JGitUtils.getStringContent(repository, tree, path.path); + IssueModel issue = JsonUtils.fromJsonString(json, IssueModel.class); + if (filter == null) { + list.add(issue); + } else { + if (filter.accept(issue)) { + list.add(issue); + } + } + } + Collections.sort(list); + return list; + } + + /** + * Retrieves the specified issue from the repository with complete changes + * history. + * + * @param repository + * @param issueId + * @return an issue, if it exists, otherwise null + */ + public static IssueModel getIssue(Repository repository, String issueId) { + RefModel issuesBranch = getIssuesBranch(repository); + if (issuesBranch == null) { + return null; + } + + if (StringUtils.isEmpty(issueId)) { + return null; + } + + // deserialize the issue model object + IssueModel issue = null; + String issuePath = getIssuePath(issueId); + RevTree tree = JGitUtils.getCommit(repository, GB_ISSUES).getTree(); + String json = JGitUtils.getStringContent(repository, tree, issuePath + "/issue.json"); + issue = JsonUtils.fromJsonString(json, IssueModel.class); + return issue; + } + + /** + * Retrieves the specified attachment from an issue. + * + * @param repository + * @param issueId + * @param filename + * @return an attachment, if found, null otherwise + */ + public static Attachment getIssueAttachment(Repository repository, String issueId, + String filename) { + RefModel issuesBranch = getIssuesBranch(repository); + if (issuesBranch == null) { + return null; + } + + if (StringUtils.isEmpty(issueId)) { + return null; + } + + // deserialize the issue model so that we have the attachment metadata + String issuePath = getIssuePath(issueId); + RevTree tree = JGitUtils.getCommit(repository, GB_ISSUES).getTree(); + String json = JGitUtils.getStringContent(repository, tree, issuePath + "/issue.json"); + IssueModel issue = JsonUtils.fromJsonString(json, IssueModel.class); + Attachment attachment = issue.getAttachment(filename); + + // attachment not found + if (attachment == null) { + return null; + } + + // retrieve the attachment content + byte[] content = JGitUtils.getByteContent(repository, tree, issuePath + "/" + filename); + attachment.content = content; + attachment.size = content.length; + return attachment; + } + + /** + * Stores an issue in the gb-issues branch of the repository. The branch is + * automatically created if it does not already exist. + * + * @param repository + * @param change + * @return true if successful + */ + public static IssueModel createIssue(Repository repository, Change change) { + RefModel issuesBranch = getIssuesBranch(repository); + if (issuesBranch == null) { + JGitUtils.createOrphanBranch(repository, "gb-issues", null); + } + change.created = new Date(); + + IssueModel issue = new IssueModel(); + issue.created = change.created; + issue.summary = change.getString(Field.Summary); + issue.description = change.getString(Field.Description); + issue.reporter = change.getString(Field.Reporter); + + if (StringUtils.isEmpty(issue.summary)) { + throw new RuntimeException("Must specify an issue summary!"); + } + if (StringUtils.isEmpty(change.getString(Field.Description))) { + throw new RuntimeException("Must specify an issue description!"); + } + if (StringUtils.isEmpty(change.getString(Field.Reporter))) { + throw new RuntimeException("Must specify an issue reporter!"); + } + + issue.id = StringUtils.getSHA1(issue.created.toString() + issue.reporter + issue.summary + + issue.description); + + String message = createChangelog('+', issue.id, change); + boolean success = commit(repository, issue, change, message); + if (success) { + return issue; + } + return null; + } + + /** + * Updates an issue in the gb-issues branch of the repository. + * + * @param repository + * @param issue + * @param change + * @return true if successful + */ + public static boolean updateIssue(Repository repository, String issueId, Change change) { + boolean success = false; + RefModel issuesBranch = getIssuesBranch(repository); + + if (issuesBranch == null) { + throw new RuntimeException("gb-issues branch does not exist!"); + } + + if (change == null) { + throw new RuntimeException("change can not be null!"); + } + + if (StringUtils.isEmpty(change.author)) { + throw new RuntimeException("must specify change.author!"); + } + + IssueModel issue = getIssue(repository, issueId); + change.created = new Date(); + + String message = createChangelog('=', issueId, change); + success = commit(repository, issue, change, message); + return success; + } + + private static String createChangelog(char type, String issueId, Change change) { + return type + " " + issueId + "\n\n" + toJson(change); + } + + /** + * + * @param repository + * @param issue + * @param change + * @param changelog + * @return + */ + private static boolean commit(Repository repository, IssueModel issue, Change change, + String changelog) { + boolean success = false; + String issuePath = getIssuePath(issue.id); + try { + issue.addChange(change); + + // serialize the issue as json + String json = toJson(issue); + + // cache the issue "files" in a map + Map files = new HashMap(); + CommitFile issueFile = new CommitFile(issuePath + "/issue.json", change.created); + issueFile.content = json.getBytes(Constants.CHARACTER_ENCODING); + files.put(issueFile.path, issueFile); + + if (change.hasAttachments()) { + for (Attachment attachment : change.attachments) { + if (!ArrayUtils.isEmpty(attachment.content)) { + CommitFile file = new CommitFile(issuePath + "/" + attachment.name, + change.created); + file.content = attachment.content; + files.put(file.path, file); + } + } + } + + ObjectId headId = repository.resolve(GB_ISSUES + "^{commit}"); + + ObjectInserter odi = repository.newObjectInserter(); + try { + // Create the in-memory index of the new/updated issue. + DirCache index = createIndex(repository, headId, files); + ObjectId indexTreeId = index.writeTree(odi); + + // Create a commit object + PersonIdent author = new PersonIdent(issue.reporter, issue.reporter + "@gitblit"); + CommitBuilder commit = new CommitBuilder(); + commit.setAuthor(author); + commit.setCommitter(author); + commit.setEncoding(Constants.CHARACTER_ENCODING); + commit.setMessage(changelog); + commit.setParentId(headId); + commit.setTreeId(indexTreeId); + + // Insert the commit into the repository + ObjectId commitId = odi.insert(commit); + odi.flush(); + + RevWalk revWalk = new RevWalk(repository); + try { + RevCommit revCommit = revWalk.parseCommit(commitId); + RefUpdate ru = repository.updateRef(GB_ISSUES); + ru.setNewObjectId(commitId); + ru.setExpectedOldObjectId(headId); + ru.setRefLogMessage("commit: " + revCommit.getShortMessage(), false); + Result rc = ru.forceUpdate(); + switch (rc) { + case NEW: + case FORCED: + case FAST_FORWARD: + success = true; + break; + case REJECTED: + case LOCK_FAILURE: + throw new ConcurrentRefUpdateException(JGitText.get().couldNotLockHEAD, + ru.getRef(), rc); + default: + throw new JGitInternalException(MessageFormat.format( + JGitText.get().updatingRefFailed, GB_ISSUES, commitId.toString(), + rc)); + } + } finally { + revWalk.release(); + } + } finally { + odi.release(); + } + } catch (Throwable t) { + t.printStackTrace(); + } + return success; + } + + private static String toJson(Object o) { + try { + // exclude the attachment content field from json serialization + Gson gson = JsonUtils.gson(new ExcludeField( + "com.gitblit.models.IssueModel$Attachment.content")); + String json = gson.toJson(o); + return json; + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + + /** + * Returns the issue path. This follows the same scheme as Git's object + * store path where the first two characters of the hash id are the root + * folder with the remaining characters as a subfolder within that folder. + * + * @param issueId + * @return the root path of the issue content on the gb-issues branch + */ + private static String getIssuePath(String issueId) { + return issueId.substring(0, 2) + "/" + issueId.substring(2); + } + + /** + * Creates an in-memory index of the issue change. + * + * @param repo + * @param headId + * @param files + * @param time + * @return an in-memory index + * @throws IOException + */ + private static DirCache createIndex(Repository repo, ObjectId headId, + Map files) throws IOException { + + DirCache inCoreIndex = DirCache.newInCore(); + DirCacheBuilder dcBuilder = inCoreIndex.builder(); + ObjectInserter inserter = repo.newObjectInserter(); + + try { + // Add the issue files to the temporary index + for (CommitFile file : files.values()) { + // create an index entry for the file + final DirCacheEntry dcEntry = new DirCacheEntry(file.path); + dcEntry.setLength(file.content.length); + dcEntry.setLastModified(file.time); + dcEntry.setFileMode(FileMode.REGULAR_FILE); + + // insert object + dcEntry.setObjectId(inserter.insert(Constants.OBJ_BLOB, file.content)); + + // add to temporary in-core index + dcBuilder.add(dcEntry); + } + + // Traverse HEAD to add all other paths + TreeWalk treeWalk = new TreeWalk(repo); + int hIdx = -1; + if (headId != null) + hIdx = treeWalk.addTree(new RevWalk(repo).parseTree(headId)); + treeWalk.setRecursive(true); + + while (treeWalk.next()) { + String path = treeWalk.getPathString(); + CanonicalTreeParser hTree = null; + if (hIdx != -1) + hTree = treeWalk.getTree(hIdx, CanonicalTreeParser.class); + if (!files.containsKey(path)) { + // add entries from HEAD for all other paths + if (hTree != null) { + // create a new DirCacheEntry with data retrieved from + // HEAD + final DirCacheEntry dcEntry = new DirCacheEntry(path); + dcEntry.setObjectId(hTree.getEntryObjectId()); + dcEntry.setFileMode(hTree.getEntryFileMode()); + + // add to temporary in-core index + dcBuilder.add(dcEntry); + } + } + } + + // release the treewalk + treeWalk.release(); + + // finish temporary in-core index used for this commit + dcBuilder.finish(); + } finally { + inserter.release(); + } + return inCoreIndex; + } + + private static class CommitFile { + String path; + long time; + byte[] content; + + CommitFile(String path, Date date) { + this.path = path; + this.time = date.getTime(); + } + } + + public static interface IssueFilter { + public abstract boolean accept(IssueModel issue); + } +} diff --git a/src/com/gitblit/utils/JGitUtils.java b/src/com/gitblit/utils/JGitUtils.java index a540c2aa..5d6011a2 100644 --- a/src/com/gitblit/utils/JGitUtils.java +++ b/src/com/gitblit/utils/JGitUtils.java @@ -24,7 +24,6 @@ import java.nio.charset.Charset; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -748,25 +747,40 @@ public class JGitUtils { } /** - * Returns the list of files in the repository that match one of the - * specified extensions. This is a CASE-SENSITIVE search. If the repository - * does not exist or is empty, an empty list is returned. + * Returns the list of files in the repository on the default branch that + * match one of the specified extensions. This is a CASE-SENSITIVE search. + * If the repository does not exist or is empty, an empty list is returned. * * @param repository * @param extensions * @return list of files in repository with a matching extension */ public static List getDocuments(Repository repository, List extensions) { + return getDocuments(repository, extensions, null); + } + + /** + * Returns the list of files in the repository in the specified commit that + * match one of the specified extensions. This is a CASE-SENSITIVE search. + * If the repository does not exist or is empty, an empty list is returned. + * + * @param repository + * @param extensions + * @param objectId + * @return list of files in repository with a matching extension + */ + public static List getDocuments(Repository repository, List extensions, + String objectId) { List list = new ArrayList(); if (!hasCommits(repository)) { return list; } - RevCommit commit = getCommit(repository, null); + RevCommit commit = getCommit(repository, objectId); final TreeWalk tw = new TreeWalk(repository); try { tw.addTree(commit.getTree()); if (extensions != null && extensions.size() > 0) { - Collection suffixFilters = new ArrayList(); + List suffixFilters = new ArrayList(); for (String extension : extensions) { if (extension.charAt(0) == '.') { suffixFilters.add(PathSuffixFilter.create("\\" + extension)); @@ -775,7 +789,12 @@ public class JGitUtils { suffixFilters.add(PathSuffixFilter.create("\\." + extension)); } } - TreeFilter filter = OrTreeFilter.create(suffixFilters); + TreeFilter filter; + if (suffixFilters.size() == 1) { + filter = suffixFilters.get(0); + } else { + filter = OrTreeFilter.create(suffixFilters); + } tw.setFilter(filter); tw.setRecursive(true); } diff --git a/src/com/gitblit/utils/JsonUtils.java b/src/com/gitblit/utils/JsonUtils.java index da9c99d2..aea46bbb 100644 --- a/src/com/gitblit/utils/JsonUtils.java +++ b/src/com/gitblit/utils/JsonUtils.java @@ -38,6 +38,8 @@ import com.gitblit.GitBlitException.UnauthorizedException; import com.gitblit.GitBlitException.UnknownRequestException; import com.gitblit.models.RepositoryModel; import com.gitblit.models.UserModel; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonDeserializationContext; @@ -108,7 +110,7 @@ public class JsonUtils { UnauthorizedException { return retrieveJson(url, type, null, null); } - + /** * Reads a gson object from the specified url. * @@ -169,10 +171,11 @@ public class JsonUtils { */ public static String retrieveJsonString(String url, String username, char[] password) throws IOException { - try { + try { URLConnection conn = ConnectionUtils.openReadConnection(url, username, password); InputStream is = conn.getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(is, ConnectionUtils.CHARSET)); + BufferedReader reader = new BufferedReader(new InputStreamReader(is, + ConnectionUtils.CHARSET)); StringBuilder json = new StringBuilder(); char[] buffer = new char[4096]; int len = 0; @@ -260,10 +263,13 @@ public class JsonUtils { // build custom gson instance with GMT date serializer/deserializer // http://code.google.com/p/google-gson/issues/detail?id=281 - private static Gson gson() { + public static Gson gson(ExclusionStrategy... strategies) { GsonBuilder builder = new GsonBuilder(); builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter()); builder.setPrettyPrinting(); + if (!ArrayUtils.isEmpty(strategies)) { + builder.setExclusionStrategies(strategies); + } return builder.create(); } @@ -296,4 +302,24 @@ public class JsonUtils { } } } + + public static class ExcludeField implements ExclusionStrategy { + + private Class c; + private String fieldName; + + public ExcludeField(String fqfn) throws SecurityException, NoSuchFieldException, + ClassNotFoundException { + this.c = Class.forName(fqfn.substring(0, fqfn.lastIndexOf("."))); + this.fieldName = fqfn.substring(fqfn.lastIndexOf(".") + 1); + } + + public boolean shouldSkipClass(Class arg0) { + return false; + } + + public boolean shouldSkipField(FieldAttributes f) { + return (f.getDeclaringClass() == c && f.getName().equals(fieldName)); + } + } } -- cgit v1.2.3 From c508386dba5dd84de1f83fef8781b06acf903f15 Mon Sep 17 00:00:00 2001 From: James Moger Date: Fri, 13 Jan 2012 08:18:12 -0500 Subject: Change build identifiers --- src/com/gitblit/Constants.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/com') diff --git a/src/com/gitblit/Constants.java b/src/com/gitblit/Constants.java index cedda6f4..a28a871a 100644 --- a/src/com/gitblit/Constants.java +++ b/src/com/gitblit/Constants.java @@ -29,11 +29,11 @@ public class Constants { // The build script extracts this exact line so be careful editing it // and only use A-Z a-z 0-9 .-_ in the string. - public static final String VERSION = "0.8.2"; + public static final String VERSION = "0.9.0-SNAPSHOT"; // The build script extracts this exact line so be careful editing it // and only use A-Z a-z 0-9 .-_ in the string. - public static final String VERSION_DATE = "2012-01-13"; + public static final String VERSION_DATE = "PENDING"; // The build script extracts this exact line so be careful editing it // and only use A-Z a-z 0-9 .-_ in the string. -- cgit v1.2.3 From 69a55934079b299740fa4679fbbd9faeb8319726 Mon Sep 17 00:00:00 2001 From: James Moger Date: Sun, 15 Jan 2012 19:07:34 -0500 Subject: More functional issues. --- src/com/gitblit/models/IssueModel.java | 305 ++++++++++++--- src/com/gitblit/utils/IssueUtils.java | 614 ++++++++++++++++++++++++------ src/com/gitblit/utils/JGitUtils.java | 2 +- tests/com/gitblit/tests/GitBlitSuite.java | 2 +- tests/com/gitblit/tests/IssuesTest.java | 165 ++++++-- 5 files changed, 884 insertions(+), 204 deletions(-) (limited to 'src/com') diff --git a/src/com/gitblit/models/IssueModel.java b/src/com/gitblit/models/IssueModel.java index 3c6d9a0b..19241c29 100644 --- a/src/com/gitblit/models/IssueModel.java +++ b/src/com/gitblit/models/IssueModel.java @@ -18,10 +18,13 @@ package com.gitblit.models; import java.io.Serializable; import java.util.ArrayList; import java.util.Date; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import com.gitblit.utils.ArrayUtils; import com.gitblit.utils.StringUtils; +import com.gitblit.utils.TimeUtils; /** * The Gitblit Issue model, its component classes, and enums. @@ -31,7 +34,7 @@ import com.gitblit.utils.StringUtils; */ public class IssueModel implements Serializable, Comparable { - private static final long serialVersionUID = 1L;; + private static final long serialVersionUID = 1L; public String id; @@ -56,7 +59,7 @@ public class IssueModel implements Serializable, Comparable { public List changes; public IssueModel() { - created = new Date(); + created = new Date((System.currentTimeMillis() / 1000) * 1000); type = Type.Defect; status = Status.New; @@ -72,15 +75,16 @@ public class IssueModel implements Serializable, Comparable { return s; } + public boolean hasLabel(String label) { + return getLabels().contains(label); + } + public List getLabels() { List list = new ArrayList(); String labels = null; for (Change change : changes) { - if (change.hasFieldChanges()) { - FieldChange field = change.getField(Field.Labels); - if (field != null) { - labels = field.value.toString(); - } + if (change.hasField(Field.Labels)) { + labels = change.getString(Field.Labels); } } if (!StringUtils.isEmpty(labels)) { @@ -89,10 +93,6 @@ public class IssueModel implements Serializable, Comparable { return list; } - public boolean hasLabel(String label) { - return getLabels().contains(label); - } - public Attachment getAttachment(String name) { Attachment attachment = null; for (Change change : changes) { @@ -106,16 +106,55 @@ public class IssueModel implements Serializable, Comparable { return attachment; } - public void addChange(Change change) { - if (changes == null) { - changes = new ArrayList(); - } + public void applyChange(Change change) { changes.add(change); + + if (change.hasFieldChanges()) { + for (FieldChange fieldChange : change.fieldChanges) { + switch (fieldChange.field) { + case Id: + id = fieldChange.value.toString(); + break; + case Type: + type = IssueModel.Type.fromObject(fieldChange.value); + break; + case Status: + status = IssueModel.Status.fromObject(fieldChange.value); + break; + case Priority: + priority = IssueModel.Priority.fromObject(fieldChange.value); + break; + case Summary: + summary = fieldChange.value.toString(); + break; + case Description: + description = fieldChange.value.toString(); + break; + case Reporter: + reporter = fieldChange.value.toString(); + break; + case Owner: + owner = fieldChange.value.toString(); + break; + case Milestone: + milestone = fieldChange.value.toString(); + break; + } + } + } } @Override public String toString() { - return summary; + StringBuilder sb = new StringBuilder(); + sb.append("issue "); + sb.append(id.substring(0, 8)); + sb.append(" (" + summary + ")\n"); + for (Change change : changes) { + sb.append(change); + sb.append('\n'); + } + return sb.toString(); } @Override @@ -135,41 +174,72 @@ public class IssueModel implements Serializable, Comparable { return id.hashCode(); } - public static class Change implements Serializable { + public static class Change implements Serializable, Comparable { private static final long serialVersionUID = 1L; - public Date created; + public final Date created; + + public final String author; - public String author; + public String id; + + public char code; public Comment comment; - public List fieldChanges; + public Set fieldChanges; - public List attachments; + public Set attachments; - public void comment(String text) { - comment = new Comment(text); + public Change(String author) { + this.created = new Date((System.currentTimeMillis() / 1000) * 1000); + this.author = author; + this.id = StringUtils.getSHA1(created.toString() + author); } public boolean hasComment() { - return comment != null; + return comment != null && !comment.deleted; + } + + public void comment(String text) { + comment = new Comment(text); + comment.id = StringUtils.getSHA1(created.toString() + author + text); } public boolean hasAttachments() { return !ArrayUtils.isEmpty(attachments); } + public void addAttachment(Attachment attachment) { + if (attachments == null) { + attachments = new LinkedHashSet(); + } + attachments.add(attachment); + } + + public Attachment getAttachment(String name) { + for (Attachment attachment : attachments) { + if (attachment.name.equalsIgnoreCase(name)) { + return attachment; + } + } + return null; + } + + public boolean hasField(Field field) { + return !StringUtils.isEmpty(getString(field)); + } + public boolean hasFieldChanges() { return !ArrayUtils.isEmpty(fieldChanges); } - public FieldChange getField(Field field) { + public Object getField(Field field) { if (fieldChanges != null) { for (FieldChange fieldChange : fieldChanges) { if (fieldChange.field == field) { - return fieldChange; + return fieldChange.value; } } } @@ -177,42 +247,74 @@ public class IssueModel implements Serializable, Comparable { } public void setField(Field field, Object value) { - FieldChange fieldChange = new FieldChange(); - fieldChange.field = field; - fieldChange.value = value; + FieldChange fieldChange = new FieldChange(field, value); if (fieldChanges == null) { - fieldChanges = new ArrayList(); + fieldChanges = new LinkedHashSet(); } fieldChanges.add(fieldChange); } public String getString(Field field) { - FieldChange fieldChange = getField(field); - if (fieldChange == null) { + Object value = getField(field); + if (value == null) { return null; } - return fieldChange.value.toString(); + return value.toString(); } - public void addAttachment(Attachment attachment) { - if (attachments == null) { - attachments = new ArrayList(); - } - attachments.add(attachment); + @Override + public int compareTo(Change c) { + return created.compareTo(c.created); } - public Attachment getAttachment(String name) { - for (Attachment attachment : attachments) { - if (attachment.name.equalsIgnoreCase(name)) { - return attachment; - } + @Override + public int hashCode() { + return id.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof Change) { + return id.equals(((Change) o).id); } - return null; + return false; } @Override public String toString() { - return created.toString() + " by " + author; + StringBuilder sb = new StringBuilder(); + sb.append(TimeUtils.timeAgo(created)); + switch (code) { + case '+': + sb.append(" created by "); + break; + default: + if (hasComment()) { + sb.append(" commented on by "); + } else { + sb.append(" changed by "); + } + } + sb.append(author).append(" - "); + if (hasComment()) { + if (comment.deleted) { + sb.append("(deleted) "); + } + sb.append(comment.text).append(" "); + } + if (hasFieldChanges()) { + switch (code) { + case '+': + break; + default: + for (FieldChange fieldChange : fieldChanges) { + sb.append("\n "); + sb.append(fieldChange); + } + break; + } + } + return sb.toString(); } } @@ -221,6 +323,9 @@ public class IssueModel implements Serializable, Comparable { private static final long serialVersionUID = 1L; public String text; + + public String id; + public boolean deleted; Comment(String text) { @@ -237,9 +342,27 @@ public class IssueModel implements Serializable, Comparable { private static final long serialVersionUID = 1L; - public Field field; + public final Field field; - public Object value; + public final Object value; + + FieldChange(Field field, Object value) { + this.field = field; + this.value = value; + } + + @Override + public int hashCode() { + return field.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof FieldChange) { + return field.equals(((FieldChange) o).field); + } + return false; + } @Override public String toString() { @@ -251,7 +374,8 @@ public class IssueModel implements Serializable, Comparable { private static final long serialVersionUID = 1L; - public String name; + public final String name; + public String id; public long size; public byte[] content; public boolean deleted; @@ -260,6 +384,19 @@ public class IssueModel implements Serializable, Comparable { this.name = name; } + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof Attachment) { + return name.equalsIgnoreCase(((Attachment) o).name); + } + return false; + } + @Override public String toString() { return name; @@ -267,20 +404,86 @@ public class IssueModel implements Serializable, Comparable { } public static enum Field { - Summary, Description, Reporter, Owner, Type, Status, Priority, Milestone, Labels; + Id, Summary, Description, Reporter, Owner, Type, Status, Priority, Milestone, Component, Labels; } public static enum Type { Defect, Enhancement, Task, Review, Other; + + public static Type fromObject(Object o) { + if (o instanceof Type) { + // cast and return + return (Type) o; + } else if (o instanceof String) { + // find by name + for (Type type : values()) { + String str = o.toString(); + if (type.toString().equalsIgnoreCase(str)) { + return type; + } + } + } else if (o instanceof Number) { + // by ordinal + int id = ((Number) o).intValue(); + if (id >= 0 && id < values().length) { + return values()[id]; + } + } + return null; + } } public static enum Priority { Low, Medium, High, Critical; + + public static Priority fromObject(Object o) { + if (o instanceof Priority) { + // cast and return + return (Priority) o; + } else if (o instanceof String) { + // find by name + for (Priority priority : values()) { + String str = o.toString(); + if (priority.toString().equalsIgnoreCase(str)) { + return priority; + } + } + } else if (o instanceof Number) { + // by ordinal + int id = ((Number) o).intValue(); + if (id >= 0 && id < values().length) { + return values()[id]; + } + } + return null; + } } public static enum Status { New, Accepted, Started, Review, Queued, Testing, Done, Fixed, WontFix, Duplicate, Invalid; + public static Status fromObject(Object o) { + if (o instanceof Status) { + // cast and return + return (Status) o; + } else if (o instanceof String) { + // find by name + for (Status status : values()) { + String str = o.toString(); + if (status.toString().equalsIgnoreCase(str)) { + return status; + } + } + } else if (o instanceof Number) { + // by ordinal + int id = ((Number) o).intValue(); + if (id >= 0 && id < values().length) { + return values()[id]; + } + } + return null; + } + public boolean atLeast(Status status) { return ordinal() >= status.ordinal(); } @@ -289,6 +492,10 @@ public class IssueModel implements Serializable, Comparable { return ordinal() > status.ordinal(); } + public boolean isClosed() { + return ordinal() >= Done.ordinal(); + } + public Status next() { switch (this) { case New: diff --git a/src/com/gitblit/utils/IssueUtils.java b/src/com/gitblit/utils/IssueUtils.java index 82170703..d0a01992 100644 --- a/src/com/gitblit/utils/IssueUtils.java +++ b/src/com/gitblit/utils/IssueUtils.java @@ -18,12 +18,15 @@ package com.gitblit.utils; import java.io.IOException; import java.text.MessageFormat; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collection; import java.util.Collections; -import java.util.Date; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import org.eclipse.jgit.JGitText; import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException; @@ -45,15 +48,21 @@ import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.AndTreeFilter; +import org.eclipse.jgit.treewalk.filter.PathFilterGroup; +import org.eclipse.jgit.treewalk.filter.TreeFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.gitblit.models.IssueModel; import com.gitblit.models.IssueModel.Attachment; import com.gitblit.models.IssueModel.Change; import com.gitblit.models.IssueModel.Field; -import com.gitblit.models.PathModel; +import com.gitblit.models.IssueModel.Status; import com.gitblit.models.RefModel; import com.gitblit.utils.JsonUtils.ExcludeField; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; /** * Utility class for reading Gitblit issues. @@ -63,8 +72,37 @@ import com.google.gson.Gson; */ public class IssueUtils { + public static interface IssueFilter { + public abstract boolean accept(IssueModel issue); + } + public static final String GB_ISSUES = "refs/heads/gb-issues"; + static final Logger LOGGER = LoggerFactory.getLogger(JGitUtils.class); + + /** + * Log an error message and exception. + * + * @param t + * @param repository + * if repository is not null it MUST be the {0} parameter in the + * pattern. + * @param pattern + * @param objects + */ + private static void error(Throwable t, Repository repository, String pattern, Object... objects) { + List parameters = new ArrayList(); + if (objects != null && objects.length > 0) { + for (Object o : objects) { + parameters.add(o); + } + } + if (repository != null) { + parameters.add(0, repository.getDirectory().getAbsolutePath()); + } + LOGGER.error(MessageFormat.format(pattern, parameters.toArray()), t); + } + /** * Returns a RefModel for the gb-issues branch in the repository. If the * branch can not be found, null is returned. @@ -77,7 +115,10 @@ public class IssueUtils { } /** - * Returns all the issues in the repository. + * Returns all the issues in the repository. Querying issues from the + * repository requires deserializing all changes for all issues. This is an + * expensive process and not recommended. Issues should be indexed by Lucene + * and queries should be executed against that index. * * @param repository * @param filter @@ -90,12 +131,85 @@ public class IssueUtils { if (issuesBranch == null) { return list; } - List paths = JGitUtils - .getDocuments(repository, Arrays.asList("json"), GB_ISSUES); - RevTree tree = JGitUtils.getCommit(repository, GB_ISSUES).getTree(); - for (PathModel path : paths) { - String json = JGitUtils.getStringContent(repository, tree, path.path); - IssueModel issue = JsonUtils.fromJsonString(json, IssueModel.class); + + // Collect the set of all issue paths + Set issuePaths = new HashSet(); + final TreeWalk tw = new TreeWalk(repository); + try { + RevCommit head = JGitUtils.getCommit(repository, GB_ISSUES); + tw.addTree(head.getTree()); + tw.setRecursive(false); + while (tw.next()) { + if (tw.getDepth() < 2 && tw.isSubtree()) { + tw.enterSubtree(); + if (tw.getDepth() == 2) { + issuePaths.add(tw.getPathString()); + } + } + } + } catch (IOException e) { + error(e, repository, "{0} failed to query issues"); + } finally { + tw.release(); + } + + // Build each issue and optionally filter out unwanted issues + + for (String issuePath : issuePaths) { + RevWalk rw = new RevWalk(repository); + try { + RevCommit start = rw.parseCommit(repository.resolve(GB_ISSUES)); + rw.markStart(start); + } catch (Exception e) { + error(e, repository, "Failed to find {1} in {0}", GB_ISSUES); + } + TreeFilter treeFilter = AndTreeFilter.create( + PathFilterGroup.createFromStrings(issuePath), TreeFilter.ANY_DIFF); + rw.setTreeFilter(treeFilter); + Iterator revlog = rw.iterator(); + + List commits = new ArrayList(); + while (revlog.hasNext()) { + commits.add(revlog.next()); + } + + // release the revwalk + rw.release(); + + if (commits.size() == 0) { + LOGGER.warn("Failed to find changes for issue " + issuePath); + continue; + } + + // sort by commit order, first commit first + Collections.reverse(commits); + + StringBuilder sb = new StringBuilder("["); + boolean first = true; + for (RevCommit commit : commits) { + if (!first) { + sb.append(','); + } + String message = commit.getFullMessage(); + // commit message is formatted: C ISSUEID\n\nJSON + // C is an single char commit code + // ISSUEID is an SHA-1 hash + String json = message.substring(43); + sb.append(json); + first = false; + } + sb.append(']'); + + // Deserialize the JSON array as a Collection, this seems + // slightly faster than deserializing each change by itself. + Collection changes = JsonUtils.fromJsonString(sb.toString(), + new TypeToken>() { + }.getType()); + + // create an issue object form the changes + IssueModel issue = buildIssue(changes, true); + + // add the issue, conditionally, to the list if (filter == null) { list.add(issue); } else { @@ -104,19 +218,36 @@ public class IssueUtils { } } } + + // sort the issues by creation Collections.sort(list); return list; } /** - * Retrieves the specified issue from the repository with complete changes - * history. + * Retrieves the specified issue from the repository with all changes + * applied to build the effective issue. * * @param repository * @param issueId * @return an issue, if it exists, otherwise null */ public static IssueModel getIssue(Repository repository, String issueId) { + return getIssue(repository, issueId, true); + } + + /** + * Retrieves the specified issue from the repository. + * + * @param repository + * @param issueId + * @param effective + * if true, the effective issue is built by processing comment + * changes, deletions, etc. if false, the raw issue is built + * without consideration for comment changes, deletions, etc. + * @return an issue, if it exists, otherwise null + */ + public static IssueModel getIssue(Repository repository, String issueId, boolean effective) { RefModel issuesBranch = getIssuesBranch(repository); if (issuesBranch == null) { return null; @@ -126,12 +257,88 @@ public class IssueUtils { return null; } - // deserialize the issue model object - IssueModel issue = null; String issuePath = getIssuePath(issueId); - RevTree tree = JGitUtils.getCommit(repository, GB_ISSUES).getTree(); - String json = JGitUtils.getStringContent(repository, tree, issuePath + "/issue.json"); - issue = JsonUtils.fromJsonString(json, IssueModel.class); + + // Collect all changes as JSON array from commit messages + List commits = JGitUtils.getRevLog(repository, GB_ISSUES, issuePath, 0, -1); + + // sort by commit order, first commit first + Collections.reverse(commits); + + StringBuilder sb = new StringBuilder("["); + boolean first = true; + for (RevCommit commit : commits) { + if (!first) { + sb.append(','); + } + String message = commit.getFullMessage(); + // commit message is formatted: C ISSUEID\n\nJSON + // C is an single char commit code + // ISSUEID is an SHA-1 hash + String json = message.substring(43); + sb.append(json); + first = false; + } + sb.append(']'); + + // Deserialize the JSON array as a Collection, this seems + // slightly faster than deserializing each change by itself. + Collection changes = JsonUtils.fromJsonString(sb.toString(), + new TypeToken>() { + }.getType()); + + // create an issue object and apply the changes to it + IssueModel issue = buildIssue(changes, effective); + return issue; + } + + /** + * Builds an issue from a set of changes. + * + * @param changes + * @param effective + * if true, the effective issue is built which accounts for + * comment changes, comment deletions, etc. if false, the raw + * issue is built. + * @return an issue + */ + private static IssueModel buildIssue(Collection changes, boolean effective) { + IssueModel issue; + if (effective) { + List effectiveChanges = new ArrayList(); + Map comments = new HashMap(); + for (Change change : changes) { + if (change.comment != null) { + if (comments.containsKey(change.comment.id)) { + Change original = comments.get(change.comment.id); + Change clone = DeepCopier.copy(original); + clone.comment.text = change.comment.text; + clone.comment.deleted = change.comment.deleted; + int idx = effectiveChanges.indexOf(original); + effectiveChanges.remove(original); + effectiveChanges.add(idx, clone); + comments.put(clone.comment.id, clone); + } else { + effectiveChanges.add(change); + comments.put(change.comment.id, change); + } + } else { + effectiveChanges.add(change); + } + } + + // effective issue + issue = new IssueModel(); + for (Change change : effectiveChanges) { + issue.applyChange(change); + } + } else { + // raw issue + issue = new IssueModel(); + for (Change change : changes) { + issue.applyChange(change); + } + } return issue; } @@ -155,10 +362,7 @@ public class IssueUtils { } // deserialize the issue model so that we have the attachment metadata - String issuePath = getIssuePath(issueId); - RevTree tree = JGitUtils.getCommit(repository, GB_ISSUES).getTree(); - String json = JGitUtils.getStringContent(repository, tree, issuePath + "/issue.json"); - IssueModel issue = JsonUtils.fromJsonString(json, IssueModel.class); + IssueModel issue = getIssue(repository, issueId, true); Attachment attachment = issue.getAttachment(filename); // attachment not found @@ -167,15 +371,21 @@ public class IssueUtils { } // retrieve the attachment content - byte[] content = JGitUtils.getByteContent(repository, tree, issuePath + "/" + filename); + String issuePath = getIssuePath(issueId); + RevTree tree = JGitUtils.getCommit(repository, GB_ISSUES).getTree(); + byte[] content = JGitUtils + .getByteContent(repository, tree, issuePath + "/" + attachment.id); attachment.content = content; attachment.size = content.length; return attachment; } /** - * Stores an issue in the gb-issues branch of the repository. The branch is - * automatically created if it does not already exist. + * Creates an issue in the gb-issues branch of the repository. The branch is + * automatically created if it does not already exist. Your change must + * include an author, summary, and description, at a minimum. If your change + * does not have those minimum requirements a RuntimeException will be + * thrown. * * @param repository * @param change @@ -186,31 +396,27 @@ public class IssueUtils { if (issuesBranch == null) { JGitUtils.createOrphanBranch(repository, "gb-issues", null); } - change.created = new Date(); - IssueModel issue = new IssueModel(); - issue.created = change.created; - issue.summary = change.getString(Field.Summary); - issue.description = change.getString(Field.Description); - issue.reporter = change.getString(Field.Reporter); - - if (StringUtils.isEmpty(issue.summary)) { - throw new RuntimeException("Must specify an issue summary!"); + if (StringUtils.isEmpty(change.author)) { + throw new RuntimeException("Must specify a change author!"); } - if (StringUtils.isEmpty(change.getString(Field.Description))) { - throw new RuntimeException("Must specify an issue description!"); + if (!change.hasField(Field.Summary)) { + throw new RuntimeException("Must specify a summary!"); } - if (StringUtils.isEmpty(change.getString(Field.Reporter))) { - throw new RuntimeException("Must specify an issue reporter!"); + if (!change.hasField(Field.Description)) { + throw new RuntimeException("Must specify a description!"); } - issue.id = StringUtils.getSHA1(issue.created.toString() + issue.reporter + issue.summary - + issue.description); + change.setField(Field.Reporter, change.author); + + String issueId = StringUtils.getSHA1(change.created.toString() + change.author + + change.getString(Field.Summary) + change.getField(Field.Description)); + change.setField(Field.Id, issueId); + change.code = '+'; - String message = createChangelog('+', issue.id, change); - boolean success = commit(repository, issue, change, message); + boolean success = commit(repository, issueId, change); if (success) { - return issue; + return getIssue(repository, issueId, false); } return null; } @@ -236,71 +442,254 @@ public class IssueUtils { } if (StringUtils.isEmpty(change.author)) { - throw new RuntimeException("must specify change.author!"); + throw new RuntimeException("must specify a change author!"); } - IssueModel issue = getIssue(repository, issueId); - change.created = new Date(); - - String message = createChangelog('=', issueId, change); - success = commit(repository, issue, change, message); + // determine update code + // default update code is '=' for a general change + change.code = '='; + if (change.hasField(Field.Status)) { + Status status = Status.fromObject(change.getField(Field.Status)); + if (status.isClosed()) { + // someone closed the issue + change.code = 'x'; + } + } + success = commit(repository, issueId, change); return success; } - private static String createChangelog(char type, String issueId, Change change) { - return type + " " + issueId + "\n\n" + toJson(change); + /** + * Deletes an issue from the repository. + * + * @param repository + * @param issueId + * @return true if successful + */ + public static boolean deleteIssue(Repository repository, String issueId, String author) { + boolean success = false; + RefModel issuesBranch = getIssuesBranch(repository); + + if (issuesBranch == null) { + throw new RuntimeException("gb-issues branch does not exist!"); + } + + if (StringUtils.isEmpty(issueId)) { + throw new RuntimeException("must specify an issue id!"); + } + + String issuePath = getIssuePath(issueId); + + String message = "- " + issueId; + try { + ObjectId headId = repository.resolve(GB_ISSUES + "^{commit}"); + ObjectInserter odi = repository.newObjectInserter(); + try { + // Create the in-memory index of the new/updated issue + DirCache index = DirCache.newInCore(); + DirCacheBuilder dcBuilder = index.builder(); + // Traverse HEAD to add all other paths + TreeWalk treeWalk = new TreeWalk(repository); + int hIdx = -1; + if (headId != null) + hIdx = treeWalk.addTree(new RevWalk(repository).parseTree(headId)); + treeWalk.setRecursive(true); + while (treeWalk.next()) { + String path = treeWalk.getPathString(); + CanonicalTreeParser hTree = null; + if (hIdx != -1) + hTree = treeWalk.getTree(hIdx, CanonicalTreeParser.class); + if (!path.startsWith(issuePath)) { + // add entries from HEAD for all other paths + if (hTree != null) { + // create a new DirCacheEntry with data retrieved + // from HEAD + final DirCacheEntry dcEntry = new DirCacheEntry(path); + dcEntry.setObjectId(hTree.getEntryObjectId()); + dcEntry.setFileMode(hTree.getEntryFileMode()); + + // add to temporary in-core index + dcBuilder.add(dcEntry); + } + } + } + + // release the treewalk + treeWalk.release(); + + // finish temporary in-core index used for this commit + dcBuilder.finish(); + + ObjectId indexTreeId = index.writeTree(odi); + + // Create a commit object + PersonIdent ident = new PersonIdent(author, "gitblit@localhost"); + CommitBuilder commit = new CommitBuilder(); + commit.setAuthor(ident); + commit.setCommitter(ident); + commit.setEncoding(Constants.CHARACTER_ENCODING); + commit.setMessage(message); + commit.setParentId(headId); + commit.setTreeId(indexTreeId); + + // Insert the commit into the repository + ObjectId commitId = odi.insert(commit); + odi.flush(); + + RevWalk revWalk = new RevWalk(repository); + try { + RevCommit revCommit = revWalk.parseCommit(commitId); + RefUpdate ru = repository.updateRef(GB_ISSUES); + ru.setNewObjectId(commitId); + ru.setExpectedOldObjectId(headId); + ru.setRefLogMessage("commit: " + revCommit.getShortMessage(), false); + Result rc = ru.forceUpdate(); + switch (rc) { + case NEW: + case FORCED: + case FAST_FORWARD: + success = true; + break; + case REJECTED: + case LOCK_FAILURE: + throw new ConcurrentRefUpdateException(JGitText.get().couldNotLockHEAD, + ru.getRef(), rc); + default: + throw new JGitInternalException(MessageFormat.format( + JGitText.get().updatingRefFailed, GB_ISSUES, commitId.toString(), + rc)); + } + } finally { + revWalk.release(); + } + } finally { + odi.release(); + } + } catch (Throwable t) { + error(t, repository, "Failed to delete issue {1} to {0}", issueId); + } + return success; } /** + * Changes the text of an issue comment. * * @param repository * @param issue * @param change - * @param changelog - * @return + * the change with the comment to change + * @param author + * the author of the revision + * @param comment + * the revised comment + * @return true, if the change was successful */ - private static boolean commit(Repository repository, IssueModel issue, Change change, - String changelog) { - boolean success = false; - String issuePath = getIssuePath(issue.id); - try { - issue.addChange(change); + public static boolean changeComment(Repository repository, IssueModel issue, Change change, + String author, String comment) { + Change revision = new Change(author); + revision.comment(comment); + revision.comment.id = change.comment.id; + return updateIssue(repository, issue.id, revision); + } - // serialize the issue as json - String json = toJson(issue); + /** + * Deletes a comment from an issue. + * + * @param repository + * @param issue + * @param change + * the change with the comment to delete + * @param author + * @return true, if the deletion was successful + */ + public static boolean deleteComment(Repository repository, IssueModel issue, Change change, + String author) { + Change deletion = new Change(author); + deletion.comment(change.comment.text); + deletion.comment.id = change.comment.id; + deletion.comment.deleted = true; + return updateIssue(repository, issue.id, deletion); + } - // cache the issue "files" in a map - Map files = new HashMap(); - CommitFile issueFile = new CommitFile(issuePath + "/issue.json", change.created); - issueFile.content = json.getBytes(Constants.CHARACTER_ENCODING); - files.put(issueFile.path, issueFile); + /** + * Commit a change to the repository. Each issue is composed on changes. + * Issues are built from applying the changes in the order they were + * committed to the repository. The changes are actually specified in the + * commit messages and not in the RevTrees which allows for clean, + * distributed merging. + * + * @param repository + * @param issue + * @param change + * @return true, if the change was committed + */ + private static boolean commit(Repository repository, String issueId, Change change) { + boolean success = false; + try { + // assign ids to new attachments + // attachments are stored by an SHA1 id if (change.hasAttachments()) { for (Attachment attachment : change.attachments) { if (!ArrayUtils.isEmpty(attachment.content)) { - CommitFile file = new CommitFile(issuePath + "/" + attachment.name, - change.created); - file.content = attachment.content; - files.put(file.path, file); + byte[] prefix = (change.created.toString() + change.author).getBytes(); + byte[] bytes = new byte[prefix.length + attachment.content.length]; + System.arraycopy(prefix, 0, bytes, 0, prefix.length); + System.arraycopy(attachment.content, 0, bytes, prefix.length, + attachment.content.length); + attachment.id = "attachment-" + StringUtils.getSHA1(bytes); } } } - ObjectId headId = repository.resolve(GB_ISSUES + "^{commit}"); + // serialize the change as json + // exclude any attachment from json serialization + Gson gson = JsonUtils.gson(new ExcludeField( + "com.gitblit.models.IssueModel$Attachment.content")); + String json = gson.toJson(change); + + // include the json change in the commit message + String issuePath = getIssuePath(issueId); + String message = change.code + " " + issueId + "\n\n" + json; + + // Create a commit file. This is required for a proper commit and + // ensures we can retrieve the commit log of the issue path. + // + // This file is NOT serialized as part of the Change object. + switch (change.code) { + case '+': { + // New Issue. + Attachment placeholder = new Attachment("issue"); + placeholder.id = placeholder.name; + placeholder.content = "DO NOT REMOVE".getBytes(Constants.CHARACTER_ENCODING); + change.addAttachment(placeholder); + break; + } + default: { + // Update Issue. + String changeId = StringUtils.getSHA1(json); + Attachment placeholder = new Attachment("change-" + changeId); + placeholder.id = placeholder.name; + placeholder.content = "REMOVABLE".getBytes(Constants.CHARACTER_ENCODING); + change.addAttachment(placeholder); + break; + } + } + ObjectId headId = repository.resolve(GB_ISSUES + "^{commit}"); ObjectInserter odi = repository.newObjectInserter(); try { - // Create the in-memory index of the new/updated issue. - DirCache index = createIndex(repository, headId, files); + // Create the in-memory index of the new/updated issue + DirCache index = createIndex(repository, headId, issuePath, change); ObjectId indexTreeId = index.writeTree(odi); // Create a commit object - PersonIdent author = new PersonIdent(issue.reporter, issue.reporter + "@gitblit"); + PersonIdent ident = new PersonIdent(change.author, "gitblit@localhost"); CommitBuilder commit = new CommitBuilder(); - commit.setAuthor(author); - commit.setCommitter(author); + commit.setAuthor(ident); + commit.setCommitter(ident); commit.setEncoding(Constants.CHARACTER_ENCODING); - commit.setMessage(changelog); + commit.setMessage(message); commit.setParentId(headId); commit.setTreeId(indexTreeId); @@ -338,23 +727,11 @@ public class IssueUtils { odi.release(); } } catch (Throwable t) { - t.printStackTrace(); + error(t, repository, "Failed to commit issue {1} to {0}", issueId); } return success; } - private static String toJson(Object o) { - try { - // exclude the attachment content field from json serialization - Gson gson = JsonUtils.gson(new ExcludeField( - "com.gitblit.models.IssueModel$Attachment.content")); - String json = gson.toJson(o); - return json; - } catch (Throwable t) { - throw new RuntimeException(t); - } - } - /** * Returns the issue path. This follows the same scheme as Git's object * store path where the first two characters of the hash id are the root @@ -372,32 +749,38 @@ public class IssueUtils { * * @param repo * @param headId - * @param files - * @param time + * @param change * @return an in-memory index * @throws IOException */ - private static DirCache createIndex(Repository repo, ObjectId headId, - Map files) throws IOException { + private static DirCache createIndex(Repository repo, ObjectId headId, String issuePath, + Change change) throws IOException { DirCache inCoreIndex = DirCache.newInCore(); DirCacheBuilder dcBuilder = inCoreIndex.builder(); ObjectInserter inserter = repo.newObjectInserter(); + Set ignorePaths = new TreeSet(); try { - // Add the issue files to the temporary index - for (CommitFile file : files.values()) { - // create an index entry for the file - final DirCacheEntry dcEntry = new DirCacheEntry(file.path); - dcEntry.setLength(file.content.length); - dcEntry.setLastModified(file.time); - dcEntry.setFileMode(FileMode.REGULAR_FILE); - - // insert object - dcEntry.setObjectId(inserter.insert(Constants.OBJ_BLOB, file.content)); - - // add to temporary in-core index - dcBuilder.add(dcEntry); + // Add any attachments to the temporary index + if (change.hasAttachments()) { + for (Attachment attachment : change.attachments) { + // build a path name for the attachment and mark as ignored + String path = issuePath + "/" + attachment.id; + ignorePaths.add(path); + + // create an index entry for this attachment + final DirCacheEntry dcEntry = new DirCacheEntry(path); + dcEntry.setLength(attachment.content.length); + dcEntry.setLastModified(change.created.getTime()); + dcEntry.setFileMode(FileMode.REGULAR_FILE); + + // insert object + dcEntry.setObjectId(inserter.insert(Constants.OBJ_BLOB, attachment.content)); + + // add to temporary in-core index + dcBuilder.add(dcEntry); + } } // Traverse HEAD to add all other paths @@ -412,7 +795,7 @@ public class IssueUtils { CanonicalTreeParser hTree = null; if (hIdx != -1) hTree = treeWalk.getTree(hIdx, CanonicalTreeParser.class); - if (!files.containsKey(path)) { + if (!ignorePaths.contains(path)) { // add entries from HEAD for all other paths if (hTree != null) { // create a new DirCacheEntry with data retrieved from @@ -437,19 +820,4 @@ public class IssueUtils { } return inCoreIndex; } - - private static class CommitFile { - String path; - long time; - byte[] content; - - CommitFile(String path, Date date) { - this.path = path; - this.time = date.getTime(); - } - } - - public static interface IssueFilter { - public abstract boolean accept(IssueModel issue); - } -} +} \ No newline at end of file diff --git a/src/com/gitblit/utils/JGitUtils.java b/src/com/gitblit/utils/JGitUtils.java index 5d6011a2..96e6bf10 100644 --- a/src/com/gitblit/utils/JGitUtils.java +++ b/src/com/gitblit/utils/JGitUtils.java @@ -1408,7 +1408,7 @@ public class JGitUtils { // Create a tree object to reference from a commit TreeFormatter tree = new TreeFormatter(); - tree.append("NEWBRANCH", FileMode.REGULAR_FILE, blobId); + tree.append(".branch", FileMode.REGULAR_FILE, blobId); ObjectId treeId = odi.insert(tree); // Create a commit object diff --git a/tests/com/gitblit/tests/GitBlitSuite.java b/tests/com/gitblit/tests/GitBlitSuite.java index 747ce1f3..9e5caf0b 100644 --- a/tests/com/gitblit/tests/GitBlitSuite.java +++ b/tests/com/gitblit/tests/GitBlitSuite.java @@ -52,7 +52,7 @@ import com.gitblit.utils.JGitUtils; ObjectCacheTest.class, UserServiceTest.class, MarkdownUtilsTest.class, JGitUtilsTest.class, SyndicationUtilsTest.class, DiffUtilsTest.class, MetricUtilsTest.class, TicgitUtilsTest.class, GitBlitTest.class, FederationTests.class, RpcTests.class, - GitServletTest.class, GroovyScriptTest.class }) + GitServletTest.class, GroovyScriptTest.class, IssuesTest.class }) public class GitBlitSuite { public static final File REPOSITORIES = new File("git"); diff --git a/tests/com/gitblit/tests/IssuesTest.java b/tests/com/gitblit/tests/IssuesTest.java index 1522ec69..26b59956 100644 --- a/tests/com/gitblit/tests/IssuesTest.java +++ b/tests/com/gitblit/tests/IssuesTest.java @@ -16,6 +16,7 @@ package com.gitblit.tests; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -30,16 +31,23 @@ import com.gitblit.models.IssueModel.Attachment; import com.gitblit.models.IssueModel.Change; import com.gitblit.models.IssueModel.Field; import com.gitblit.models.IssueModel.Priority; +import com.gitblit.models.IssueModel.Status; import com.gitblit.utils.IssueUtils; import com.gitblit.utils.IssueUtils.IssueFilter; +/** + * Tests the mechanics of distributed issue management on the gb-issues branch. + * + * @author James Moger + * + */ public class IssuesTest { @Test - public void testInsertion() throws Exception { + public void testCreation() throws Exception { Repository repository = GitBlitSuite.getIssuesTestRepository(); // create and insert the issue - Change c1 = newChange("Test issue " + Long.toHexString(System.currentTimeMillis())); + Change c1 = newChange("testCreation() " + Long.toHexString(System.currentTimeMillis())); IssueModel issue = IssueUtils.createIssue(repository, c1); assertNotNull(issue.id); @@ -47,68 +55,165 @@ public class IssuesTest { IssueModel constructed = IssueUtils.getIssue(repository, issue.id); compare(issue, constructed); - // add a note and update - Change c2 = new Change(); - c2.author = "dave"; - c2.comment("yeah, this is working"); + assertEquals(1, constructed.changes.size()); + } + + @Test + public void testUpdates() throws Exception { + Repository repository = GitBlitSuite.getIssuesTestRepository(); + // C1: create the issue + Change c1 = newChange("testUpdates() " + Long.toHexString(System.currentTimeMillis())); + IssueModel issue = IssueUtils.createIssue(repository, c1); + assertNotNull(issue.id); + + IssueModel constructed = IssueUtils.getIssue(repository, issue.id); + compare(issue, constructed); + + // C2: set owner + Change c2 = new Change("C2"); + c2.comment("I'll fix this"); + c2.setField(Field.Owner, c2.author); assertTrue(IssueUtils.updateIssue(repository, issue.id, c2)); + constructed = IssueUtils.getIssue(repository, issue.id); + assertEquals(2, constructed.changes.size()); + assertEquals(c2.author, constructed.owner); + + // C3: add a note + Change c3 = new Change("C3"); + c3.comment("yeah, this is working"); + assertTrue(IssueUtils.updateIssue(repository, issue.id, c3)); + constructed = IssueUtils.getIssue(repository, issue.id); + assertEquals(3, constructed.changes.size()); + + // C4: add attachment + Change c4 = new Change("C4"); + Attachment a = newAttachment(); + c4.addAttachment(a); + assertTrue(IssueUtils.updateIssue(repository, issue.id, c4)); + + Attachment a1 = IssueUtils.getIssueAttachment(repository, issue.id, a.name); + assertEquals(a.content.length, a1.content.length); + assertTrue(Arrays.areEqual(a.content, a1.content)); + + // C5: close the issue + Change c5 = new Change("C5"); + c5.comment("closing issue"); + c5.setField(Field.Status, Status.Fixed); + assertTrue(IssueUtils.updateIssue(repository, issue.id, c5)); // retrieve issue again constructed = IssueUtils.getIssue(repository, issue.id); - assertEquals(2, constructed.changes.size()); + assertEquals(5, constructed.changes.size()); + assertTrue(constructed.status.isClosed()); - Attachment a = IssueUtils.getIssueAttachment(repository, issue.id, "test.txt"); repository.close(); - - assertEquals(10, a.content.length); - assertTrue(Arrays.areEqual(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, a.content)); } @Test public void testQuery() throws Exception { Repository repository = GitBlitSuite.getIssuesTestRepository(); - List list = IssueUtils.getIssues(repository, null); - List list2 = IssueUtils.getIssues(repository, new IssueFilter() { - boolean hasFirst = false; + List allIssues = IssueUtils.getIssues(repository, null); + + List openIssues = IssueUtils.getIssues(repository, new IssueFilter() { @Override public boolean accept(IssueModel issue) { - if (!hasFirst) { - hasFirst = true; - return true; - } - return false; + return !issue.status.isClosed(); } }); + + List closedIssues = IssueUtils.getIssues(repository, new IssueFilter() { + @Override + public boolean accept(IssueModel issue) { + return issue.status.isClosed(); + } + }); + + repository.close(); + assertTrue(allIssues.size() > 0); + assertEquals(1, openIssues.size()); + assertEquals(1, closedIssues.size()); + } + + @Test + public void testDelete() throws Exception { + Repository repository = GitBlitSuite.getIssuesTestRepository(); + List allIssues = IssueUtils.getIssues(repository, null); + // delete all issues + for (IssueModel issue : allIssues) { + assertTrue(IssueUtils.deleteIssue(repository, issue.id, "D")); + } + repository.close(); + } + + @Test + public void testChangeComment() throws Exception { + Repository repository = GitBlitSuite.getIssuesTestRepository(); + // C1: create the issue + Change c1 = newChange("testChangeComment() " + Long.toHexString(System.currentTimeMillis())); + IssueModel issue = IssueUtils.createIssue(repository, c1); + assertNotNull(issue.id); + assertTrue(issue.changes.get(0).hasComment()); + + assertTrue(IssueUtils.changeComment(repository, issue, c1, "E1", "I changed the comment")); + issue = IssueUtils.getIssue(repository, issue.id); + assertTrue(issue.changes.get(0).hasComment()); + assertEquals("I changed the comment", issue.changes.get(0).comment.text); + + assertTrue(IssueUtils.deleteIssue(repository, issue.id, "D")); + + repository.close(); + } + + @Test + public void testDeleteComment() throws Exception { + Repository repository = GitBlitSuite.getIssuesTestRepository(); + // C1: create the issue + Change c1 = newChange("testDeleteComment() " + Long.toHexString(System.currentTimeMillis())); + IssueModel issue = IssueUtils.createIssue(repository, c1); + assertNotNull(issue.id); + assertTrue(issue.changes.get(0).hasComment()); + + assertTrue(IssueUtils.deleteComment(repository, issue, c1, "D1")); + issue = IssueUtils.getIssue(repository, issue.id); + assertEquals(1, issue.changes.size()); + assertFalse(issue.changes.get(0).hasComment()); + + issue = IssueUtils.getIssue(repository, issue.id, false); + assertEquals(2, issue.changes.size()); + assertTrue(issue.changes.get(0).hasComment()); + assertFalse(issue.changes.get(1).hasComment()); + + assertTrue(IssueUtils.deleteIssue(repository, issue.id, "D")); + repository.close(); - assertTrue(list.size() > 0); - assertEquals(1, list2.size()); } private Change newChange(String summary) { - Change change = new Change(); - change.setField(Field.Reporter, "james"); - change.setField(Field.Owner, "dave"); + Change change = new Change("C1"); change.setField(Field.Summary, summary); change.setField(Field.Description, "this is my description"); change.setField(Field.Priority, Priority.High); change.setField(Field.Labels, "helpdesk"); change.comment("my comment"); - - Attachment attachment = new Attachment("test.txt"); - attachment.content = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - change.addAttachment(attachment); - return change; } + private Attachment newAttachment() { + Attachment attachment = new Attachment(Long.toHexString(System.currentTimeMillis()) + + ".txt"); + attachment.content = new byte[] { 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, + 0x4a }; + return attachment; + } + private void compare(IssueModel issue, IssueModel constructed) { assertEquals(issue.id, constructed.id); assertEquals(issue.reporter, constructed.reporter); assertEquals(issue.owner, constructed.owner); - assertEquals(issue.created.getTime() / 1000, constructed.created.getTime() / 1000); assertEquals(issue.summary, constructed.summary); assertEquals(issue.description, constructed.description); + assertEquals(issue.created, constructed.created); assertTrue(issue.hasLabel("helpdesk")); } -- cgit v1.2.3 From 06ff61c04db32abc7ba4bbaeee604482c343152e Mon Sep 17 00:00:00 2001 From: James Moger Date: Tue, 24 Jan 2012 22:49:53 -0500 Subject: Fairly functional Lucene backend for commits, blobs, and issues --- src/com/gitblit/build/Build.java | 6 + src/com/gitblit/models/IssueModel.java | 10 + src/com/gitblit/models/SearchResult.java | 42 +++ src/com/gitblit/utils/IssueUtils.java | 2 +- src/com/gitblit/utils/LuceneUtils.java | 506 +++++++++++++++++++++++++++ tests/com/gitblit/tests/GitBlitSuite.java | 2 +- tests/com/gitblit/tests/IssuesTest.java | 36 ++ tests/com/gitblit/tests/LuceneUtilsTest.java | 109 ++++++ 8 files changed, 711 insertions(+), 2 deletions(-) create mode 100644 src/com/gitblit/models/SearchResult.java create mode 100644 src/com/gitblit/utils/LuceneUtils.java create mode 100644 tests/com/gitblit/tests/LuceneUtilsTest.java (limited to 'src/com') diff --git a/src/com/gitblit/build/Build.java b/src/com/gitblit/build/Build.java index 682adcad..b539cac6 100644 --- a/src/com/gitblit/build/Build.java +++ b/src/com/gitblit/build/Build.java @@ -90,6 +90,7 @@ public class Build { downloadFromApache(MavenObject.GSON, BuildType.RUNTIME); downloadFromApache(MavenObject.MAIL, BuildType.RUNTIME); downloadFromApache(MavenObject.GROOVY, BuildType.RUNTIME); + downloadFromApache(MavenObject.LUCENE, BuildType.RUNTIME); downloadFromEclipse(MavenObject.JGIT, BuildType.RUNTIME); downloadFromEclipse(MavenObject.JGIT_HTTP, BuildType.RUNTIME); @@ -116,6 +117,7 @@ public class Build { downloadFromApache(MavenObject.GSON, BuildType.COMPILETIME); downloadFromApache(MavenObject.MAIL, BuildType.COMPILETIME); downloadFromApache(MavenObject.GROOVY, BuildType.COMPILETIME); + downloadFromApache(MavenObject.LUCENE, BuildType.COMPILETIME); downloadFromEclipse(MavenObject.JGIT, BuildType.COMPILETIME); downloadFromEclipse(MavenObject.JGIT_HTTP, BuildType.COMPILETIME); @@ -501,6 +503,10 @@ public class Build { "1.8.5", 6143000, 2290000, 4608000, "3be3914c49ca7d8e8afb29a7772a74c30a1f1b28", "1435cc8c90e3a91e5fee7bb53e83aad96e93aeb7", "5a214b52286523f9e2a4b5fed526506c763fa6f1"); + public static final MavenObject LUCENE = new MavenObject("lucene", "org/apache/lucene", "lucene-core", + "3.5.0", 1470000, 1347000, 3608000, "90ff0731fafb05c01fee4f2247140d56e9c30a3b", + "0757113199f9c8c18c678c96d61c2c4160b9baa6", "19f8e80e5e7f6ec88a41d4f63495994692e31bf1"); + public final String name; public final String group; public final String artifact; diff --git a/src/com/gitblit/models/IssueModel.java b/src/com/gitblit/models/IssueModel.java index 19241c29..246b29bf 100644 --- a/src/com/gitblit/models/IssueModel.java +++ b/src/com/gitblit/models/IssueModel.java @@ -106,6 +106,16 @@ public class IssueModel implements Serializable, Comparable { return attachment; } + public List getAttachments() { + List list = new ArrayList(); + for (Change change : changes) { + if (change.hasAttachments()) { + list.addAll(change.attachments); + } + } + return list; + } + public void applyChange(Change change) { changes.add(change); diff --git a/src/com/gitblit/models/SearchResult.java b/src/com/gitblit/models/SearchResult.java new file mode 100644 index 00000000..44207f08 --- /dev/null +++ b/src/com/gitblit/models/SearchResult.java @@ -0,0 +1,42 @@ +package com.gitblit.models; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +import com.gitblit.utils.LuceneUtils.ObjectType; + +/** + * Model class that represents a search result. + * + * @author James Moger + * + */ +public class SearchResult implements Serializable { + + private static final long serialVersionUID = 1L; + + public float score; + + public Date date; + + public String author; + + public String committer; + + public String summary; + + public String id; + + public List labels; + + public ObjectType type; + + public SearchResult() { + } + + @Override + public String toString() { + return type.name() + ": " + id; + } +} \ No newline at end of file diff --git a/src/com/gitblit/utils/IssueUtils.java b/src/com/gitblit/utils/IssueUtils.java index d0a01992..eb3b347b 100644 --- a/src/com/gitblit/utils/IssueUtils.java +++ b/src/com/gitblit/utils/IssueUtils.java @@ -740,7 +740,7 @@ public class IssueUtils { * @param issueId * @return the root path of the issue content on the gb-issues branch */ - private static String getIssuePath(String issueId) { + static String getIssuePath(String issueId) { return issueId.substring(0, 2) + "/" + issueId.substring(2); } diff --git a/src/com/gitblit/utils/LuceneUtils.java b/src/com/gitblit/utils/LuceneUtils.java new file mode 100644 index 00000000..4ca72f0a --- /dev/null +++ b/src/com/gitblit/utils/LuceneUtils.java @@ -0,0 +1,506 @@ +package com.gitblit.utils; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.DateTools; +import org.apache.lucene.document.DateTools.Resolution; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.Version; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.TreeWalk; + +import com.gitblit.models.IssueModel; +import com.gitblit.models.IssueModel.Attachment; +import com.gitblit.models.RefModel; +import com.gitblit.models.SearchResult; + +/** + * A collection of utility methods for indexing and querying a Lucene repository + * index. + * + * @author James Moger + * + */ +public class LuceneUtils { + + /** + * The types of objects that can be indexed and queried. + */ + public static enum ObjectType { + commit, blob, issue; + + static ObjectType fromName(String name) { + for (ObjectType value : values()) { + if (value.name().equals(name)) { + return value; + } + } + return null; + } + } + + private static final Version LUCENE_VERSION = Version.LUCENE_35; + + private static final String FIELD_OBJECT_TYPE = "type"; + private static final String FIELD_OBJECT_ID = "id"; + private static final String FIELD_REPOSITORY = "repository"; + private static final String FIELD_SUMMARY = "summary"; + private static final String FIELD_CONTENT = "content"; + private static final String FIELD_AUTHOR = "author"; + private static final String FIELD_COMMITTER = "committer"; + private static final String FIELD_DATE = "date"; + private static final String FIELD_LABEL = "label"; + private static final String FIELD_ATTACHMENT = "attachment"; + + private static Set excludes = new TreeSet(Arrays.asList("7z", "arc", "arj", + "bin", "bmp", "dll", "doc", "docx", "exe", "gif", "gz", "jar", "jpg", "lib", "lzh", + "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls", "xlsx", "zip")); + + private static final Map SEARCHERS = new ConcurrentHashMap(); + private static final Map WRITERS = new ConcurrentHashMap(); + + /** + * Deletes the Lucene index for the specified repository. + * + * @param repository + * @return true, if successful + */ + public static boolean deleteIndex(Repository repository) { + try { + File luceneIndex = new File(repository.getDirectory(), "lucene"); + if (luceneIndex.exists()) { + org.eclipse.jgit.util.FileUtils.delete(luceneIndex, + org.eclipse.jgit.util.FileUtils.RECURSIVE); + } + return true; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * This completely indexes the repository and will destroy any existing + * index. + * + * @param repository + * @return true if the indexing has succeeded + */ + public static boolean index(Repository repository) { + try { + Set indexedCommits = new TreeSet(); + IndexWriter writer = getIndexWriter(repository, true); + // build a quick lookup of tags + Map> tags = new HashMap>(); + for (RefModel tag : JGitUtils.getTags(repository, false, -1)) { + if (!tags.containsKey(tag.getObjectId())) { + tags.put(tag.getReferencedObjectId().getName(), new ArrayList()); + } + tags.get(tag.getReferencedObjectId().getName()).add(tag.displayName); + } + + // walk through each branch + List branches = JGitUtils.getLocalBranches(repository, true, -1); + for (RefModel branch : branches) { + RevWalk revWalk = new RevWalk(repository); + RevCommit rev = revWalk.parseCommit(branch.getObjectId()); + + // index the blob contents of the tree + ByteArrayOutputStream os = new ByteArrayOutputStream(); + byte[] tmp = new byte[32767]; + TreeWalk treeWalk = new TreeWalk(repository); + treeWalk.addTree(rev.getTree()); + treeWalk.setRecursive(true); + String revDate = DateTools.timeToString(rev.getCommitTime() * 1000L, + Resolution.MINUTE); + while (treeWalk.next()) { + Document doc = new Document(); + doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES, + Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_OBJECT_ID, treeWalk.getPathString(), Store.YES, + Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO)); + doc.add(new Field(FIELD_AUTHOR, rev.getAuthorIdent().getName(), Store.YES, + Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_COMMITTER, rev.getCommitterIdent().getName(), + Store.YES, Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_LABEL, branch.getName(), Store.YES, Index.ANALYZED)); + + // determine extension to compare to the extension + // blacklist + String ext = null; + String name = treeWalk.getPathString().toLowerCase(); + if (name.indexOf('.') > -1) { + ext = name.substring(name.lastIndexOf('.') + 1); + } + + if (StringUtils.isEmpty(ext) || !excludes.contains(ext)) { + // read the blob content + ObjectId entid = treeWalk.getObjectId(0); + FileMode entmode = treeWalk.getFileMode(0); + RevObject ro = revWalk.lookupAny(entid, entmode.getObjectType()); + revWalk.parseBody(ro); + ObjectLoader ldr = repository.open(ro.getId(), Constants.OBJ_BLOB); + InputStream in = ldr.openStream(); + os.reset(); + int n = 0; + while ((n = in.read(tmp)) > 0) { + os.write(tmp, 0, n); + } + in.close(); + byte[] content = os.toByteArray(); + String str = new String(content, "UTF-8"); + doc.add(new Field(FIELD_CONTENT, str, Store.NO, Index.ANALYZED)); + writer.addDocument(doc); + } + } + + os.close(); + treeWalk.release(); + + // index the head commit object + String head = rev.getId().getName(); + if (indexedCommits.add(head)) { + Document doc = createDocument(rev, tags.get(head)); + writer.addDocument(doc); + } + + // traverse the log and index the previous commit objects + revWalk.markStart(rev); + while ((rev = revWalk.next()) != null) { + String hash = rev.getId().getName(); + if (indexedCommits.add(hash)) { + Document doc = createDocument(rev, tags.get(hash)); + writer.addDocument(doc); + } + } + + // finished + revWalk.dispose(); + } + + // this repository has a gb-issues branch, index all issues + if (IssueUtils.getIssuesBranch(repository) != null) { + List issues = IssueUtils.getIssues(repository, null); + for (IssueModel issue : issues) { + Document doc = createDocument(issue); + writer.addDocument(doc); + } + } + + // commit all changes and reset the searcher + resetIndexSearcher(repository); + writer.commit(); + return true; + } catch (Exception e) { + e.printStackTrace(); + } + return false; + } + + /** + * Incrementally update the index with the specified commit for the + * repository. + * + * @param repository + * @param commit + * @return true, if successful + */ + public static boolean index(Repository repository, RevCommit commit) { + try { + Document doc = createDocument(commit, null); + return index(repository, doc); + } catch (Exception e) { + e.printStackTrace(); + } + return false; + } + + /** + * Incrementally update the index with the specified issue for the + * repository. + * + * @param repository + * @param issue + * @param reindex + * if true, the old index entry for this issue will be deleted. + * This is only appropriate for pre-existing/indexed issues. + * @return true, if successful + */ + public static boolean index(Repository repository, IssueModel issue, boolean reindex) { + try { + Document doc = createDocument(issue); + if (reindex) { + // delete the old issue from the index, if exists + IndexWriter writer = getIndexWriter(repository, false); + writer.deleteDocuments(new Term(FIELD_OBJECT_TYPE, ObjectType.issue.name()), + new Term(FIELD_OBJECT_ID, String.valueOf(issue.id))); + writer.commit(); + } + return index(repository, doc); + } catch (Exception e) { + e.printStackTrace(); + } + return false; + } + + /** + * Creates a Lucene document from an issue. + * + * @param issue + * @return a Lucene document + */ + private static Document createDocument(IssueModel issue) { + Document doc = new Document(); + doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.issue.name(), Store.YES, + Field.Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_DATE, DateTools.dateToString(issue.created, Resolution.MINUTE), + Store.YES, Field.Index.NO)); + doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.NOT_ANALYZED_NO_NORMS)); + List attachments = new ArrayList(); + for (Attachment attachment : issue.getAttachments()) { + attachments.add(attachment.name.toLowerCase()); + } + doc.add(new Field(FIELD_ATTACHMENT, StringUtils.flattenStrings(attachments), Store.YES, + Index.ANALYZED)); + doc.add(new Field(FIELD_SUMMARY, issue.summary, Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_CONTENT, issue.toString(), Store.NO, Index.ANALYZED)); + doc.add(new Field(FIELD_LABEL, StringUtils.flattenStrings(issue.getLabels()), Store.YES, + Index.ANALYZED)); + return doc; + } + + /** + * Creates a Lucene document for a commit + * + * @param commit + * @param tags + * @return a Lucene document + */ + private static Document createDocument(RevCommit commit, List tags) { + Document doc = new Document(); + doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.commit.name(), Store.YES, + Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_OBJECT_ID, commit.getName(), Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_DATE, DateTools.timeToString(commit.getCommitTime() * 1000L, + Resolution.MINUTE), Store.YES, Index.NO)); + doc.add(new Field(FIELD_AUTHOR, commit.getCommitterIdent().getName(), Store.YES, + Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), Store.YES, Index.ANALYZED)); + doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), Store.NO, Index.ANALYZED)); + if (!ArrayUtils.isEmpty(tags)) { + if (!ArrayUtils.isEmpty(tags)) { + doc.add(new Field(FIELD_LABEL, StringUtils.flattenStrings(tags), Store.YES, + Index.ANALYZED)); + } + } + return doc; + } + + /** + * Incrementally index an object for the repository. + * + * @param repository + * @param doc + * @return true, if successful + */ + private static boolean index(Repository repository, Document doc) { + try { + IndexWriter writer = getIndexWriter(repository, false); + writer.addDocument(doc); + resetIndexSearcher(repository); + writer.commit(); + return true; + } catch (Exception e) { + e.printStackTrace(); + } + return false; + } + + private static SearchResult createSearchResult(Document doc, float score) throws ParseException { + SearchResult result = new SearchResult(); + result.score = score; + result.date = DateTools.stringToDate(doc.get(FIELD_DATE)); + result.summary = doc.get(FIELD_SUMMARY); + result.author = doc.get(FIELD_AUTHOR); + result.committer = doc.get(FIELD_COMMITTER); + result.type = ObjectType.fromName(doc.get(FIELD_OBJECT_TYPE)); + result.id = doc.get(FIELD_OBJECT_ID); + if (doc.get(FIELD_LABEL) != null) { + result.labels = StringUtils.getStringsFromValue(doc.get(FIELD_LABEL)); + } + return result; + } + + private static void resetIndexSearcher(Repository repository) throws IOException { + IndexSearcher searcher = SEARCHERS.get(repository.getDirectory()); + if (searcher != null) { + SEARCHERS.remove(repository.getDirectory()); + searcher.close(); + } + } + + /** + * Gets an index searcher for the repository. + * + * @param repository + * @return + * @throws IOException + */ + private static IndexSearcher getIndexSearcher(Repository repository) throws IOException { + IndexSearcher searcher = SEARCHERS.get(repository.getDirectory()); + if (searcher == null) { + IndexWriter writer = getIndexWriter(repository, false); + searcher = new IndexSearcher(IndexReader.open(writer, true)); + SEARCHERS.put(repository.getDirectory(), searcher); + } + return searcher; + } + + /** + * Gets an index writer for the repository. The index will be created if it + * does not already exist or if forceCreate is specified. + * + * @param repository + * @param forceCreate + * @return an IndexWriter + * @throws IOException + */ + private static IndexWriter getIndexWriter(Repository repository, boolean forceCreate) + throws IOException { + IndexWriter indexWriter = WRITERS.get(repository.getDirectory()); + File indexFolder = new File(repository.getDirectory(), "lucene"); + Directory directory = FSDirectory.open(indexFolder); + if (forceCreate || !indexFolder.exists()) { + // if the writer is going to blow away the existing index and create + // a new one then it should not be cached. instead, close any open + // writer, create a new one, and return. + if (indexWriter != null) { + indexWriter.close(); + indexWriter = null; + WRITERS.remove(repository.getDirectory()); + } + indexFolder.mkdirs(); + IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer( + LUCENE_VERSION)); + config.setOpenMode(OpenMode.CREATE); + IndexWriter writer = new IndexWriter(directory, config); + writer.close(); + } + + if (indexWriter == null) { + IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer( + LUCENE_VERSION)); + config.setOpenMode(OpenMode.APPEND); + indexWriter = new IndexWriter(directory, config); + WRITERS.put(repository.getDirectory(), indexWriter); + } + return indexWriter; + } + + /** + * Search the repository for the given text or query + * + * @param repository + * @param text + * @return a list of SearchResults + */ + public static List search(Repository repository, String text) { + if (StringUtils.isEmpty(text)) { + return null; + } + Set results = new HashSet(); + StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION); + try { + // default search checks summary and content + BooleanQuery query = new BooleanQuery(); + QueryParser qp; + qp = new QueryParser(LUCENE_VERSION, FIELD_SUMMARY, analyzer); + qp.setAllowLeadingWildcard(true); + query.add(qp.parse(text), Occur.SHOULD); + + qp = new QueryParser(LUCENE_VERSION, FIELD_CONTENT, analyzer); + qp.setAllowLeadingWildcard(true); + query.add(qp.parse(text), Occur.SHOULD); + + IndexSearcher searcher = getIndexSearcher(repository); + Query rewrittenQuery = searcher.rewrite(query); + + TopScoreDocCollector collector = TopScoreDocCollector.create(200, true); + searcher.search(rewrittenQuery, collector); + ScoreDoc[] hits = collector.topDocs().scoreDocs; + for (int i = 0; i < hits.length; i++) { + int docId = hits[i].doc; + Document doc = searcher.doc(docId); + SearchResult result = createSearchResult(doc, hits[i].score); + results.add(result); + } + } catch (Exception e) { + e.printStackTrace(); + } + return new ArrayList(results); + } + + /** + * Close all the index writers and searchers + */ + public static void close() { + // close writers + for (File file : WRITERS.keySet()) { + try { + WRITERS.get(file).close(true); + } catch (Throwable t) { + t.printStackTrace(); + } + } + WRITERS.clear(); + + // close searchers + for (File file : SEARCHERS.keySet()) { + try { + SEARCHERS.get(file).close(); + } catch (Throwable t) { + t.printStackTrace(); + } + } + SEARCHERS.clear(); + } +} diff --git a/tests/com/gitblit/tests/GitBlitSuite.java b/tests/com/gitblit/tests/GitBlitSuite.java index 9e5caf0b..8fac212c 100644 --- a/tests/com/gitblit/tests/GitBlitSuite.java +++ b/tests/com/gitblit/tests/GitBlitSuite.java @@ -52,7 +52,7 @@ import com.gitblit.utils.JGitUtils; ObjectCacheTest.class, UserServiceTest.class, MarkdownUtilsTest.class, JGitUtilsTest.class, SyndicationUtilsTest.class, DiffUtilsTest.class, MetricUtilsTest.class, TicgitUtilsTest.class, GitBlitTest.class, FederationTests.class, RpcTests.class, - GitServletTest.class, GroovyScriptTest.class, IssuesTest.class }) + GitServletTest.class, GroovyScriptTest.class, LuceneUtilsTest.class, IssuesTest.class }) public class GitBlitSuite { public static final File REPOSITORIES = new File("git"); diff --git a/tests/com/gitblit/tests/IssuesTest.java b/tests/com/gitblit/tests/IssuesTest.java index 26b59956..c475c466 100644 --- a/tests/com/gitblit/tests/IssuesTest.java +++ b/tests/com/gitblit/tests/IssuesTest.java @@ -32,8 +32,10 @@ import com.gitblit.models.IssueModel.Change; import com.gitblit.models.IssueModel.Field; import com.gitblit.models.IssueModel.Priority; import com.gitblit.models.IssueModel.Status; +import com.gitblit.models.SearchResult; import com.gitblit.utils.IssueUtils; import com.gitblit.utils.IssueUtils.IssueFilter; +import com.gitblit.utils.LuceneUtils; /** * Tests the mechanics of distributed issue management on the gb-issues branch. @@ -135,6 +137,40 @@ public class IssuesTest { assertEquals(1, closedIssues.size()); } + @Test + public void testLuceneIndexAndQuery() throws Exception { + Repository repository = GitBlitSuite.getIssuesTestRepository(); + LuceneUtils.deleteIndex(repository); + List allIssues = IssueUtils.getIssues(repository, null); + assertTrue(allIssues.size() > 0); + for (IssueModel issue : allIssues) { + LuceneUtils.index(repository, issue, false); + } + List hits = LuceneUtils.search(repository, "working"); + assertTrue(hits.size() > 0); + + // reindex an issue + IssueModel issue = allIssues.get(0); + Change change = new Change("reindex"); + change.comment("this is a test of reindexing an issue"); + IssueUtils.updateIssue(repository, issue.id, change); + issue = IssueUtils.getIssue(repository, issue.id); + LuceneUtils.index(repository, issue, true); + + LuceneUtils.close(); + repository.close(); + } + + @Test + public void testLuceneQuery() throws Exception { + Repository repository = GitBlitSuite.getIssuesTestRepository(); + List hits = LuceneUtils.search(repository, "working"); + LuceneUtils.close(); + repository.close(); + assertTrue(hits.size() > 0); + } + + @Test public void testDelete() throws Exception { Repository repository = GitBlitSuite.getIssuesTestRepository(); diff --git a/tests/com/gitblit/tests/LuceneUtilsTest.java b/tests/com/gitblit/tests/LuceneUtilsTest.java new file mode 100644 index 00000000..648c8128 --- /dev/null +++ b/tests/com/gitblit/tests/LuceneUtilsTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2012 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.tests; + +import static org.junit.Assert.assertEquals; + +import java.util.List; + +import org.eclipse.jgit.lib.Repository; +import org.junit.Test; + +import com.gitblit.models.SearchResult; +import com.gitblit.utils.LuceneUtils; + +/** + * Tests Lucene indexing and querying. + * + * @author James Moger + * + */ +public class LuceneUtilsTest { + + @Test + public void testFullIndex() throws Exception { + // reindex helloworld + Repository repository = GitBlitSuite.getHelloworldRepository(); + LuceneUtils.index(repository); + repository.close(); + + // reindex theoretical physics + repository = GitBlitSuite.getTheoreticalPhysicsRepository(); + LuceneUtils.index(repository); + repository.close(); + + // reindex bluez-gnome + repository = GitBlitSuite.getBluezGnomeRepository(); + LuceneUtils.index(repository); + repository.close(); + + LuceneUtils.close(); + } + + @Test + public void testQuery() throws Exception { + // 2 occurrences on the master branch + Repository repository = GitBlitSuite.getHelloworldRepository(); + List results = LuceneUtils.search(repository, "ada"); + assertEquals(2, results.size()); + + // author test + results = LuceneUtils.search(repository, "author: tinogomes"); + assertEquals(2, results.size()); + + repository.close(); + // blob test + results = LuceneUtils.search(repository, "type: blob AND \"import std.stdio\""); + assertEquals(1, results.size()); + assertEquals("d.D", results.get(0).id); + + // 1 occurrence on the gh-pages branch + repository = GitBlitSuite.getTheoreticalPhysicsRepository(); + results = LuceneUtils.search(repository, "\"add the .nojekyll file\""); + assertEquals(1, results.size()); + assertEquals("Ondrej Certik", results.get(0).author); + assertEquals("2648c0c98f2101180715b4d432fc58d0e21a51d7", results.get(0).id); + + // tag test + results = LuceneUtils.search(repository, "\"qft split\""); + assertEquals(1, results.size()); + assertEquals("Ondrej Certik", results.get(0).author); + assertEquals("57c4f26f157ece24b02f4f10f5f68db1d2ce7ff5", results.get(0).id); + assertEquals("[1st-edition]", results.get(0).labels.toString()); + + results = LuceneUtils.search(repository, "type:blob AND \"src/intro.rst\""); + assertEquals(4, results.size()); + + // hash id tests + results = LuceneUtils.search(repository, "id:57c4f26f157ece24b02f4f10f5f68db1d2ce7ff5"); + assertEquals(1, results.size()); + + results = LuceneUtils.search(repository, "id:57c4f26f157*"); + assertEquals(1, results.size()); + + repository.close(); + + // annotated tag test + repository = GitBlitSuite.getBluezGnomeRepository(); + results = LuceneUtils.search(repository, "\"release 1.8\""); + assertEquals(1, results.size()); + assertEquals("[1.8]", results.get(0).labels.toString()); + + repository.close(); + + LuceneUtils.close(); + } +} \ No newline at end of file -- cgit v1.2.3 From 35a71b925779d7c6c8f861f1b31b16a0d197b206 Mon Sep 17 00:00:00 2001 From: James Moger Date: Fri, 17 Feb 2012 18:42:35 -0500 Subject: Fixed date bugs on IssueModel --- src/com/gitblit/models/IssueModel.java | 7 ++++++- src/com/gitblit/utils/JsonUtils.java | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'src/com') diff --git a/src/com/gitblit/models/IssueModel.java b/src/com/gitblit/models/IssueModel.java index 246b29bf..3c191e24 100644 --- a/src/com/gitblit/models/IssueModel.java +++ b/src/com/gitblit/models/IssueModel.java @@ -59,7 +59,8 @@ public class IssueModel implements Serializable, Comparable { public List changes; public IssueModel() { - created = new Date((System.currentTimeMillis() / 1000) * 1000); + // the first applied change set the date appropriately + created = new Date(0); type = Type.Defect; status = Status.New; @@ -117,6 +118,10 @@ public class IssueModel implements Serializable, Comparable { } public void applyChange(Change change) { + if (changes.size() == 0) { + // first change created the issue + created = change.created; + } changes.add(change); if (change.hasFieldChanges()) { diff --git a/src/com/gitblit/utils/JsonUtils.java b/src/com/gitblit/utils/JsonUtils.java index aea46bbb..bc9a1e00 100644 --- a/src/com/gitblit/utils/JsonUtils.java +++ b/src/com/gitblit/utils/JsonUtils.java @@ -295,7 +295,8 @@ public class JsonUtils { JsonDeserializationContext jsonDeserializationContext) { try { synchronized (dateFormat) { - return dateFormat.parse(jsonElement.getAsString()); + Date date = dateFormat.parse(jsonElement.getAsString()); + return new Date((date.getTime() / 1000) * 1000); } } catch (ParseException e) { throw new JsonSyntaxException(jsonElement.getAsString(), e); -- cgit v1.2.3 From 36ee2307ea083cfe8994b13a3023bc779b2b23ec Mon Sep 17 00:00:00 2001 From: James Moger Date: Fri, 17 Feb 2012 18:43:40 -0500 Subject: Implemented multi-repository Lucene search --- src/com/gitblit/models/SearchResult.java | 4 +- src/com/gitblit/utils/LuceneUtils.java | 87 +++++++++++++++++++++++----- tests/com/gitblit/tests/IssuesTest.java | 4 +- tests/com/gitblit/tests/LuceneUtilsTest.java | 27 ++++++--- 4 files changed, 96 insertions(+), 26 deletions(-) (limited to 'src/com') diff --git a/src/com/gitblit/models/SearchResult.java b/src/com/gitblit/models/SearchResult.java index 44207f08..4a03a70b 100644 --- a/src/com/gitblit/models/SearchResult.java +++ b/src/com/gitblit/models/SearchResult.java @@ -25,6 +25,8 @@ public class SearchResult implements Serializable { public String committer; public String summary; + + public String repository; public String id; @@ -37,6 +39,6 @@ public class SearchResult implements Serializable { @Override public String toString() { - return type.name() + ": " + id; + return score + " : " + type.name() + " : " + repository + " : " + id; } } \ No newline at end of file diff --git a/src/com/gitblit/utils/LuceneUtils.java b/src/com/gitblit/utils/LuceneUtils.java index 4ca72f0a..483537d0 100644 --- a/src/com/gitblit/utils/LuceneUtils.java +++ b/src/com/gitblit/utils/LuceneUtils.java @@ -8,7 +8,7 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -26,6 +26,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.Term; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.search.BooleanClause.Occur; @@ -90,13 +91,32 @@ public class LuceneUtils { private static final String FIELD_LABEL = "label"; private static final String FIELD_ATTACHMENT = "attachment"; - private static Set excludes = new TreeSet(Arrays.asList("7z", "arc", "arj", - "bin", "bmp", "dll", "doc", "docx", "exe", "gif", "gz", "jar", "jpg", "lib", "lzh", - "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls", "xlsx", "zip")); + private static Set excludedExtensions = new TreeSet( + Arrays.asList("7z", "arc", "arj", "bin", "bmp", "dll", "doc", + "docx", "exe", "gif", "gz", "jar", "jpg", "lib", "lzh", + "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls", + "xlsx", "zip")); + + private static Set excludedBranches = new TreeSet( + Arrays.asList("/refs/heads/gb-issues")); private static final Map SEARCHERS = new ConcurrentHashMap(); private static final Map WRITERS = new ConcurrentHashMap(); + /** + * Returns the name of the repository. + * + * @param repository + * @return the repository name + */ + private static String getName(Repository repository) { + if (repository.isBare()) { + return repository.getDirectory().getName(); + } else { + return repository.getDirectory().getParentFile().getName(); + } + } + /** * Deletes the Lucene index for the specified repository. * @@ -125,6 +145,7 @@ public class LuceneUtils { */ public static boolean index(Repository repository) { try { + String repositoryName = getName(repository); Set indexedCommits = new TreeSet(); IndexWriter writer = getIndexWriter(repository, true); // build a quick lookup of tags @@ -139,6 +160,9 @@ public class LuceneUtils { // walk through each branch List branches = JGitUtils.getLocalBranches(repository, true, -1); for (RefModel branch : branches) { + if (excludedBranches.contains(branch.getName())) { + continue; + } RevWalk revWalk = new RevWalk(repository); RevCommit rev = revWalk.parseCommit(branch.getObjectId()); @@ -154,6 +178,8 @@ public class LuceneUtils { Document doc = new Document(); doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES, Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, + Index.NOT_ANALYZED)); doc.add(new Field(FIELD_OBJECT_ID, treeWalk.getPathString(), Store.YES, Index.NOT_ANALYZED)); doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO)); @@ -171,7 +197,7 @@ public class LuceneUtils { ext = name.substring(name.lastIndexOf('.') + 1); } - if (StringUtils.isEmpty(ext) || !excludes.contains(ext)) { + if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) { // read the blob content ObjectId entid = treeWalk.getObjectId(0); FileMode entmode = treeWalk.getFileMode(0); @@ -199,6 +225,8 @@ public class LuceneUtils { String head = rev.getId().getName(); if (indexedCommits.add(head)) { Document doc = createDocument(rev, tags.get(head)); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, + Index.NOT_ANALYZED)); writer.addDocument(doc); } @@ -208,6 +236,8 @@ public class LuceneUtils { String hash = rev.getId().getName(); if (indexedCommits.add(hash)) { Document doc = createDocument(rev, tags.get(hash)); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, + Index.NOT_ANALYZED)); writer.addDocument(doc); } } @@ -221,6 +251,8 @@ public class LuceneUtils { List issues = IssueUtils.getIssues(repository, null); for (IssueModel issue : issues) { Document doc = createDocument(issue); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, + Index.NOT_ANALYZED)); writer.addDocument(doc); } } @@ -344,6 +376,9 @@ public class LuceneUtils { */ private static boolean index(Repository repository, Document doc) { try { + String repositoryName = getName(repository); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, + Index.NOT_ANALYZED)); IndexWriter writer = getIndexWriter(repository, false); writer.addDocument(doc); resetIndexSearcher(repository); @@ -363,6 +398,7 @@ public class LuceneUtils { result.author = doc.get(FIELD_AUTHOR); result.committer = doc.get(FIELD_COMMITTER); result.type = ObjectType.fromName(doc.get(FIELD_OBJECT_TYPE)); + result.repository = doc.get(FIELD_REPOSITORY); result.id = doc.get(FIELD_OBJECT_ID); if (doc.get(FIELD_LABEL) != null) { result.labels = StringUtils.getStringsFromValue(doc.get(FIELD_LABEL)); @@ -437,17 +473,27 @@ public class LuceneUtils { } /** - * Search the repository for the given text or query + * Searches the specified repositories for the given text or query * - * @param repository * @param text - * @return a list of SearchResults + * if the text is null or empty, null is returned + * @param maximumHits + * the maximum number of hits to collect + * @param repositories + * a list of repositories to search. if no repositories are + * specified null is returned. + * @return a list of SearchResults in order from highest to the lowest score + * */ - public static List search(Repository repository, String text) { + public static List search(String text, int maximumHits, + Repository... repositories) { if (StringUtils.isEmpty(text)) { return null; } - Set results = new HashSet(); + if (repositories.length == 0) { + return null; + } + Set results = new LinkedHashSet(); StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION); try { // default search checks summary and content @@ -461,10 +507,23 @@ public class LuceneUtils { qp.setAllowLeadingWildcard(true); query.add(qp.parse(text), Occur.SHOULD); - IndexSearcher searcher = getIndexSearcher(repository); + IndexSearcher searcher; + if (repositories.length == 1) { + // single repository search + searcher = getIndexSearcher(repositories[0]); + } else { + // multiple repository search + List readers = new ArrayList(); + for (Repository repository : repositories) { + IndexSearcher repositoryIndex = getIndexSearcher(repository); + readers.add(repositoryIndex.getIndexReader()); + } + IndexReader [] rdrs = readers.toArray(new IndexReader[readers.size()]); + MultiReader reader = new MultiReader(rdrs); + searcher = new IndexSearcher(reader); + } Query rewrittenQuery = searcher.rewrite(query); - - TopScoreDocCollector collector = TopScoreDocCollector.create(200, true); + TopScoreDocCollector collector = TopScoreDocCollector.create(maximumHits, true); searcher.search(rewrittenQuery, collector); ScoreDoc[] hits = collector.topDocs().scoreDocs; for (int i = 0; i < hits.length; i++) { @@ -477,7 +536,7 @@ public class LuceneUtils { e.printStackTrace(); } return new ArrayList(results); - } + } /** * Close all the index writers and searchers diff --git a/tests/com/gitblit/tests/IssuesTest.java b/tests/com/gitblit/tests/IssuesTest.java index c475c466..a5d487d8 100644 --- a/tests/com/gitblit/tests/IssuesTest.java +++ b/tests/com/gitblit/tests/IssuesTest.java @@ -146,7 +146,7 @@ public class IssuesTest { for (IssueModel issue : allIssues) { LuceneUtils.index(repository, issue, false); } - List hits = LuceneUtils.search(repository, "working"); + List hits = LuceneUtils.search("working", 10, repository); assertTrue(hits.size() > 0); // reindex an issue @@ -164,7 +164,7 @@ public class IssuesTest { @Test public void testLuceneQuery() throws Exception { Repository repository = GitBlitSuite.getIssuesTestRepository(); - List hits = LuceneUtils.search(repository, "working"); + List hits = LuceneUtils.search("working", 10, repository); LuceneUtils.close(); repository.close(); assertTrue(hits.size() > 0); diff --git a/tests/com/gitblit/tests/LuceneUtilsTest.java b/tests/com/gitblit/tests/LuceneUtilsTest.java index 648c8128..a5446218 100644 --- a/tests/com/gitblit/tests/LuceneUtilsTest.java +++ b/tests/com/gitblit/tests/LuceneUtilsTest.java @@ -57,48 +57,48 @@ public class LuceneUtilsTest { public void testQuery() throws Exception { // 2 occurrences on the master branch Repository repository = GitBlitSuite.getHelloworldRepository(); - List results = LuceneUtils.search(repository, "ada"); + List results = LuceneUtils.search("ada", 10, repository); assertEquals(2, results.size()); // author test - results = LuceneUtils.search(repository, "author: tinogomes"); + results = LuceneUtils.search("author: tinogomes", 10, repository); assertEquals(2, results.size()); repository.close(); // blob test - results = LuceneUtils.search(repository, "type: blob AND \"import std.stdio\""); + results = LuceneUtils.search("type: blob AND \"import std.stdio\"", 10, repository); assertEquals(1, results.size()); assertEquals("d.D", results.get(0).id); // 1 occurrence on the gh-pages branch repository = GitBlitSuite.getTheoreticalPhysicsRepository(); - results = LuceneUtils.search(repository, "\"add the .nojekyll file\""); + results = LuceneUtils.search("\"add the .nojekyll file\"", 10, repository); assertEquals(1, results.size()); assertEquals("Ondrej Certik", results.get(0).author); assertEquals("2648c0c98f2101180715b4d432fc58d0e21a51d7", results.get(0).id); // tag test - results = LuceneUtils.search(repository, "\"qft split\""); + results = LuceneUtils.search("\"qft split\"", 10, repository); assertEquals(1, results.size()); assertEquals("Ondrej Certik", results.get(0).author); assertEquals("57c4f26f157ece24b02f4f10f5f68db1d2ce7ff5", results.get(0).id); assertEquals("[1st-edition]", results.get(0).labels.toString()); - results = LuceneUtils.search(repository, "type:blob AND \"src/intro.rst\""); + results = LuceneUtils.search("type:blob AND \"src/intro.rst\"", 10, repository); assertEquals(4, results.size()); // hash id tests - results = LuceneUtils.search(repository, "id:57c4f26f157ece24b02f4f10f5f68db1d2ce7ff5"); + results = LuceneUtils.search("id:57c4f26f157ece24b02f4f10f5f68db1d2ce7ff5", 10, repository); assertEquals(1, results.size()); - results = LuceneUtils.search(repository, "id:57c4f26f157*"); + results = LuceneUtils.search("id:57c4f26f157*", 10, repository); assertEquals(1, results.size()); repository.close(); // annotated tag test repository = GitBlitSuite.getBluezGnomeRepository(); - results = LuceneUtils.search(repository, "\"release 1.8\""); + results = LuceneUtils.search("\"release 1.8\"", 10, repository); assertEquals(1, results.size()); assertEquals("[1.8]", results.get(0).labels.toString()); @@ -106,4 +106,13 @@ public class LuceneUtilsTest { LuceneUtils.close(); } + + @Test + public void testMultiSearch() throws Exception { + List results = LuceneUtils.search("test", 10, + GitBlitSuite.getHelloworldRepository(), + GitBlitSuite.getBluezGnomeRepository()); + LuceneUtils.close(); + assertEquals(10, results.size()); + } } \ No newline at end of file -- cgit v1.2.3 From fb15c2c3a50aa4fe33b6bdfc9f16d759336caa6c Mon Sep 17 00:00:00 2001 From: James Moger Date: Sun, 19 Feb 2012 15:27:53 -0500 Subject: Properly index blobs on a branch --- src/com/gitblit/models/SearchResult.java | 4 +- src/com/gitblit/utils/LuceneUtils.java | 74 +++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) (limited to 'src/com') diff --git a/src/com/gitblit/models/SearchResult.java b/src/com/gitblit/models/SearchResult.java index 4a03a70b..2fa0db42 100644 --- a/src/com/gitblit/models/SearchResult.java +++ b/src/com/gitblit/models/SearchResult.java @@ -27,6 +27,8 @@ public class SearchResult implements Serializable { public String summary; public String repository; + + public String branch; public String id; @@ -39,6 +41,6 @@ public class SearchResult implements Serializable { @Override public String toString() { - return score + " : " + type.name() + " : " + repository + " : " + id; + return score + " : " + type.name() + " : " + repository + " : " + id + " (" + branch + ")"; } } \ No newline at end of file diff --git a/src/com/gitblit/utils/LuceneUtils.java b/src/com/gitblit/utils/LuceneUtils.java index 483537d0..738382a4 100644 --- a/src/com/gitblit/utils/LuceneUtils.java +++ b/src/com/gitblit/utils/LuceneUtils.java @@ -38,6 +38,7 @@ import org.apache.lucene.search.TopScoreDocCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Version; +import org.eclipse.jgit.diff.DiffEntry.ChangeType; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; @@ -50,6 +51,7 @@ import org.eclipse.jgit.treewalk.TreeWalk; import com.gitblit.models.IssueModel; import com.gitblit.models.IssueModel.Attachment; +import com.gitblit.models.PathModel.PathChangeModel; import com.gitblit.models.RefModel; import com.gitblit.models.SearchResult; @@ -82,6 +84,7 @@ public class LuceneUtils { private static final String FIELD_OBJECT_TYPE = "type"; private static final String FIELD_OBJECT_ID = "id"; + private static final String FIELD_BRANCH = "branch"; private static final String FIELD_REPOSITORY = "repository"; private static final String FIELD_SUMMARY = "summary"; private static final String FIELD_CONTENT = "content"; @@ -163,6 +166,7 @@ public class LuceneUtils { if (excludedBranches.contains(branch.getName())) { continue; } + String branchName = branch.getName(); RevWalk revWalk = new RevWalk(repository); RevCommit rev = revWalk.parseCommit(branch.getObjectId()); @@ -180,6 +184,8 @@ public class LuceneUtils { Index.NOT_ANALYZED_NO_NORMS)); doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, + Index.NOT_ANALYZED)); doc.add(new Field(FIELD_OBJECT_ID, treeWalk.getPathString(), Store.YES, Index.NOT_ANALYZED)); doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO)); @@ -227,6 +233,8 @@ public class LuceneUtils { Document doc = createDocument(rev, tags.get(head)); doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, + Index.NOT_ANALYZED)); writer.addDocument(doc); } @@ -238,6 +246,8 @@ public class LuceneUtils { Document doc = createDocument(rev, tags.get(hash)); doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, + Index.NOT_ANALYZED)); writer.addDocument(doc); } } @@ -272,11 +282,69 @@ public class LuceneUtils { * repository. * * @param repository + * @param branch + * the fully qualified branch name (e.g. refs/heads/master) * @param commit * @return true, if successful */ - public static boolean index(Repository repository, RevCommit commit) { - try { + public static boolean index(Repository repository, String branch, RevCommit commit) { + try { + if (excludedBranches.contains(branch)) { + if (IssueUtils.GB_ISSUES.equals(branch)) { + // index an issue + String issueId = commit.getShortMessage().substring(2).trim(); + IssueModel issue = IssueUtils.getIssue(repository, issueId); + return index(repository, issue, true); + } + return false; + } + List changedPaths = JGitUtils.getFilesInCommit(repository, commit); + String repositoryName = getName(repository); + String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L, + Resolution.MINUTE); + IndexWriter writer = getIndexWriter(repository, false); + for (PathChangeModel path : changedPaths) { + // delete the indexed blob + writer.deleteDocuments(new Term(FIELD_OBJECT_TYPE, ObjectType.blob.name()), + new Term(FIELD_BRANCH, branch), + new Term(FIELD_OBJECT_ID, path.path)); + + // re-index the blob + if (!ChangeType.DELETE.equals(path.changeType)) { + Document doc = new Document(); + doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES, + Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, + Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES, + Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO)); + doc.add(new Field(FIELD_AUTHOR, commit.getAuthorIdent().getName(), Store.YES, + Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_COMMITTER, commit.getCommitterIdent().getName(), + Store.YES, Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field(FIELD_LABEL, branch, Store.YES, Index.ANALYZED)); + + // determine extension to compare to the extension + // blacklist + String ext = null; + String name = path.name.toLowerCase(); + if (name.indexOf('.') > -1) { + ext = name.substring(name.lastIndexOf('.') + 1); + } + + if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) { + // read the blob content + String str = JGitUtils.getStringContent(repository, + commit.getTree(), path.path); + doc.add(new Field(FIELD_CONTENT, str, Store.NO, Index.ANALYZED)); + writer.addDocument(doc); + } + } + } + writer.commit(); + Document doc = createDocument(commit, null); return index(repository, doc); } catch (Exception e) { @@ -324,6 +392,7 @@ public class LuceneUtils { doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.issue.name(), Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.NOT_ANALYZED)); doc.add(new Field(FIELD_DATE, DateTools.dateToString(issue.created, Resolution.MINUTE), Store.YES, Field.Index.NO)); doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.NOT_ANALYZED_NO_NORMS)); @@ -399,6 +468,7 @@ public class LuceneUtils { result.committer = doc.get(FIELD_COMMITTER); result.type = ObjectType.fromName(doc.get(FIELD_OBJECT_TYPE)); result.repository = doc.get(FIELD_REPOSITORY); + result.branch = doc.get(FIELD_BRANCH); result.id = doc.get(FIELD_OBJECT_ID); if (doc.get(FIELD_LABEL) != null) { result.labels = StringUtils.getStringsFromValue(doc.get(FIELD_LABEL)); -- cgit v1.2.3