Draft integration of Lucene search mechanism and web ui

distrib/gitblit.properties

@@ -91,6 +91,26 @@ groovy.preReceiveScripts =
 # SINCE 0.8.0
 groovy.postReceiveScripts =
 
+# If true, a Lucene index will be generated and maintained for each repository.
+# Lucene search replaces brute-force Git repository traversal.
+#
+# SINCE 0.9.0
+# RESTART REQUIRED
+lucene.enable  = false
+
+# If *lucene.pollingMode* = true, Gitblit will periodically check all repositories
+# for branch updates.
+# If *lucene.pollingMode* = false, repositories will only be indexed on pushes
+# to Gitblit.
+#
+# Regardless of this setting, Gitblit will check all repositories for branch
+# updates 1 minute after startup. Indexes will automatically be built for any
+# repository that is missing its index  or if an index version change is detected.
+#
+# SINCE 0.9.0
+# RESTART REQUIRED
+lucene.pollingMode = false
+
 #
 # Authentication Settings
 #

resources/gitblit.css

@@ -266,6 +266,18 @@ img.gravatar {
     padding: 2px;
 }
 
+div.searchResult {
+	padding:5px;
+}
+
+div.searchResult .summary {
+	font-weight: bold;
+}
+
+div.searchResult .branch {
+	color: #008000;
+}
+
 div.header, div.commitHeader, table.repositories th {
 	background-color:#e0e0e0;
 	background-repeat:repeat-x;

src/com/gitblit/GitBlit.java

@@ -1824,8 +1824,8 @@ public class GitBlit implements ServletContextListener {
 		}
 		luceneExecutor = new LuceneExecutor(settings);
 		if (luceneExecutor.isReady()) {
-			logger.info("Lucene executor is scheduled to process the repository queue every 10 minutes.");
-			scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 10, TimeUnit.MINUTES);
+			logger.info("Lucene executor is scheduled to process the repository queue every 2 minutes.");
+			scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 2, TimeUnit.MINUTES);
 		} else {
 			logger.warn("Lucene executor is disabled.");
 		}

src/com/gitblit/LuceneExecutor.java

@@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory;
 import com.gitblit.models.RepositoryModel;
 import com.gitblit.utils.JGitUtils;
 import com.gitblit.utils.LuceneUtils;
+import com.gitblit.utils.LuceneUtils.IndexResult;
 
 /**
  * The Lucene executor handles indexing repositories synchronously and
@@ -53,8 +54,8 @@ public class LuceneExecutor implements Runnable {
 
 	public LuceneExecutor(IStoredSettings settings) {
 		this.settings = settings;
-		this.isLuceneEnabled = settings.getBoolean("lucene.enableLucene", false);
-		this.isPollingMode = settings.getBoolean("lucene.pollingMode", false);
+		this.isLuceneEnabled = settings.getBoolean(Keys.lucene.enable, false);
+		this.isPollingMode = settings.getBoolean(Keys.lucene.pollingMode, false);
 	}
 
 	/**
@@ -144,11 +145,14 @@ public class LuceneExecutor implements Runnable {
 				if (LuceneUtils.shouldReindex(repository)) {
 					// (re)build the entire index
 					long start = System.currentTimeMillis();
-					boolean success = LuceneUtils.reindex(repository);
+					IndexResult result = LuceneUtils.reindex(repository);
 					long duration = System.currentTimeMillis() - start;
-					if (success) {
-						String msg = "Built {0} Lucene index in {1} msecs";
-						logger.info(MessageFormat.format(msg, repositoryName, duration));
+					if (result.success) {
+						if (result.commitCount > 0) {
+							String msg = "Built {0} Lucene index from {1} commits in {2} msecs";
+							logger.info(MessageFormat.format(msg, repositoryName,
+									result.commitCount, duration));
+						}
 					} else {
 						String msg = "Could not build {0} Lucene index!";
 						logger.error(MessageFormat.format(msg, repositoryName));
@@ -156,11 +160,14 @@ public class LuceneExecutor implements Runnable {
 				} else {
 					// update the index with latest commits
 					long start = System.currentTimeMillis();
-					boolean success = LuceneUtils.updateIndex(repository);
+					IndexResult result = LuceneUtils.updateIndex(repository);
 					long duration = System.currentTimeMillis() - start;
-					if (success) {
-						String msg = "Updated {0} Lucene index in {1} msecs";
-						logger.info(MessageFormat.format(msg, repositoryName, duration));
+					if (result.success) {
+						if (result.commitCount > 0) {
+							String msg = "Updated {0} Lucene index with {1} commits in {2} msecs";
+							logger.info(MessageFormat.format(msg, repositoryName,
+									result.commitCount, duration));
+						}
 					} else {
 						String msg = "Could not update {0} Lucene index!";
 						logger.error(MessageFormat.format(msg, repositoryName));

src/com/gitblit/utils/LuceneUtils.java

@@ -52,6 +52,7 @@ import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.eclipse.jgit.treewalk.TreeWalk;
 import org.eclipse.jgit.util.FS;
 
+import com.gitblit.GitBlit;
 import com.gitblit.models.IssueModel;
 import com.gitblit.models.IssueModel.Attachment;
 import com.gitblit.models.PathModel.PathChangeModel;
@@ -121,10 +122,13 @@ public class LuceneUtils {
 	 * @return the repository name
 	 */
 	private static String getName(Repository repository) {
+		String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath();
 		if (repository.isBare()) {
-			return repository.getDirectory().getName();
+			return StringUtils.getRelativePath(rootPath, repository.getDirectory()
+					.getAbsolutePath());
 		} else {
-			return repository.getDirectory().getParentFile().getName();
+			return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile()
+					.getAbsolutePath());
 		}
 	}
 
@@ -198,11 +202,12 @@ public class LuceneUtils {
 	 * index.
 	 * 
 	 * @param repository
-	 * @return true if the indexing has succeeded
+	 * @return IndexResult
 	 */
-	public static boolean reindex(Repository repository) {
+	public static IndexResult reindex(Repository repository) {
+		IndexResult result = new IndexResult();
 		if (!LuceneUtils.deleteIndex(repository)) {
-			return false;
+			return result;
 		}
 		try {
 			String repositoryName = getName(repository);
@@ -300,6 +305,7 @@ public class LuceneUtils {
 							Index.NOT_ANALYZED));
 					doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));
 					writer.addDocument(doc);
+					result.commitCount += 1;
 				}
 
 				// traverse the log and index the previous commit objects
@@ -312,6 +318,7 @@ public class LuceneUtils {
 								Index.NOT_ANALYZED));
 						doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));
 						writer.addDocument(doc);
+						result.commitCount += 1;
 					}
 				}
 
@@ -335,11 +342,11 @@ public class LuceneUtils {
 			config.save();
 			resetIndexSearcher(repository);
 			writer.commit();
-			return true;
+			result.success = true;
 		} catch (Exception e) {
 			e.printStackTrace();
 		}
-		return false;
+		return result;
 	}
 
 	/**
@@ -453,9 +460,10 @@ public class LuceneUtils {
 	 * Updates a repository index incrementally from the last indexed commits.
 	 * 
 	 * @param repository
+	 * @return IndexResult
 	 */
-	public static boolean updateIndex(Repository repository) {
-		boolean success = false;
+	public static IndexResult updateIndex(Repository repository) {
+		IndexResult result = new IndexResult();
 		try {
 			FileBasedConfig config = getConfig(repository);
 			config.load();
@@ -475,13 +483,13 @@ public class LuceneUtils {
 
 			// detect branch deletion
 			// first assume all branches are deleted and then remove each
-			// existing branch from deletedBranches during indexing			
+			// existing branch from deletedBranches during indexing
 			Set<String> deletedBranches = new TreeSet<String>();
 			for (String alias : config.getNames(CONF_ALIAS)) {
 				String branch = config.getString(CONF_ALIAS, null, alias);
 				deletedBranches.add(branch);
 			}
-			
+
 			// walk through each branches
 			List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1);
 			for (RefModel branch : branches) {
@@ -490,7 +498,7 @@ public class LuceneUtils {
 				// remove this branch from the deletedBranches set
 				deletedBranches.remove(branchName);
 
-				// determine last commit				
+				// determine last commit
 				String keyName = getBranchKey(branchName);
 				String lastCommit = config.getString(CONF_BRANCH, null, keyName);
 
@@ -507,6 +515,7 @@ public class LuceneUtils {
 				Collections.reverse(revs);
 				for (RevCommit commit : revs) {
 					index(repository, branchName, commit);
+					result.commitCount += 1;
 				}
 
 				// update the config
@@ -515,7 +524,7 @@ public class LuceneUtils {
 				config.setString(CONF_BRANCH, null, keyName, branch.getObjectId().getName());
 				config.save();
 			}
-			
+
 			// the deletedBranches set will normally be empty by this point
 			// unless a branch really was deleted and no longer exists
 			if (deletedBranches.size() > 0) {
@@ -525,11 +534,11 @@ public class LuceneUtils {
 					writer.commit();
 				}
 			}
-			success = true;
+			result.success = true;
 		} catch (Throwable t) {
 			t.printStackTrace();
 		}
-		return success;
+		return result;
 	}
 
 	/**
@@ -782,4 +791,9 @@ public class LuceneUtils {
 		}
 		SEARCHERS.clear();
 	}
+
+	public static class IndexResult {
+		public boolean success;
+		public int commitCount;
+	}
 }

src/com/gitblit/wicket/pages/RootPage.java

@@ -101,6 +101,9 @@ public abstract class RootPage extends BasePage {
 		pages.add(new PageRegistration("gb.repositories", RepositoriesPage.class,
 				getRootPageParameters()));
 		pages.add(new PageRegistration("gb.activity", ActivityPage.class, getRootPageParameters()));
+		if (GitBlit.getBoolean(Keys.lucene.enable, false)) {
+			pages.add(new PageRegistration("gb.search", LucenePage.class));
+		}
 		if (showAdmin) {
 			pages.add(new PageRegistration("gb.users", UsersPage.class));
 		}