Draft integration of Lucene search mechanism and web ui

6 files changed, 83 insertions, 27 deletions

diff --git a/distrib/gitblit.properties b/distrib/gitblit.properties
index 2ac6598f..ede5f596 100644
--- a/distrib/gitblit.properties
+++ b/distrib/gitblit.properties
@@ -91,6 +91,26 @@ groovy.preReceiveScripts =
 # SINCE 0.8.0

 groovy.postReceiveScripts =

 

+# If true, a Lucene index will be generated and maintained for each repository.

+# Lucene search replaces brute-force Git repository traversal.

+#

+# SINCE 0.9.0

+# RESTART REQUIRED

+lucene.enable  = false

+

+# If *lucene.pollingMode* = true, Gitblit will periodically check all repositories

+# for branch updates.

+# If *lucene.pollingMode* = false, repositories will only be indexed on pushes

+# to Gitblit.

+#

+# Regardless of this setting, Gitblit will check all repositories for branch

+# updates 1 minute after startup. Indexes will automatically be built for any

+# repository that is missing its index  or if an index version change is detected.

+#

+# SINCE 0.9.0

+# RESTART REQUIRED

+lucene.pollingMode = false

+

 #

 # Authentication Settings

 #

diff --git a/resources/gitblit.css b/resources/gitblit.css
index 6ad45fb2..bfa1c202 100644
--- a/resources/gitblit.css
+++ b/resources/gitblit.css
@@ -266,6 +266,18 @@ img.gravatar {
     padding: 2px;

 }

 

+div.searchResult {

+	padding:5px;

+}

+

+div.searchResult .summary {

+	font-weight: bold;

+}

+

+div.searchResult .branch {

+	color: #008000;

+}

+

 div.header, div.commitHeader, table.repositories th {

 	background-color:#e0e0e0;

 	background-repeat:repeat-x;

diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java
index a7bfad4f..f6691dc6 100644
--- a/src/com/gitblit/GitBlit.java
+++ b/src/com/gitblit/GitBlit.java
@@ -1824,8 +1824,8 @@ public class GitBlit implements ServletContextListener {
 		}

 		luceneExecutor = new LuceneExecutor(settings);

 		if (luceneExecutor.isReady()) {

-			logger.info("Lucene executor is scheduled to process the repository queue every 10 minutes.");

-			scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 10, TimeUnit.MINUTES);

+			logger.info("Lucene executor is scheduled to process the repository queue every 2 minutes.");

+			scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 2, TimeUnit.MINUTES);

 		} else {

 			logger.warn("Lucene executor is disabled.");

 		}

diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java
index e4624efc..b4e5134a 100644
--- a/src/com/gitblit/LuceneExecutor.java
+++ b/src/com/gitblit/LuceneExecutor.java
@@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory;
 import com.gitblit.models.RepositoryModel;

 import com.gitblit.utils.JGitUtils;

 import com.gitblit.utils.LuceneUtils;

+import com.gitblit.utils.LuceneUtils.IndexResult;

 

 /**

  * The Lucene executor handles indexing repositories synchronously and

@@ -53,8 +54,8 @@ public class LuceneExecutor implements Runnable {
 

 	public LuceneExecutor(IStoredSettings settings) {

 		this.settings = settings;

-		this.isLuceneEnabled = settings.getBoolean("lucene.enableLucene", false);

-		this.isPollingMode = settings.getBoolean("lucene.pollingMode", false);

+		this.isLuceneEnabled = settings.getBoolean(Keys.lucene.enable, false);

+		this.isPollingMode = settings.getBoolean(Keys.lucene.pollingMode, false);

 	}

 

 	/**

@@ -144,11 +145,14 @@ public class LuceneExecutor implements Runnable {
 				if (LuceneUtils.shouldReindex(repository)) {

 					// (re)build the entire index

 					long start = System.currentTimeMillis();

-					boolean success = LuceneUtils.reindex(repository);

+					IndexResult result = LuceneUtils.reindex(repository);

 					long duration = System.currentTimeMillis() - start;

-					if (success) {

-						String msg = "Built {0} Lucene index in {1} msecs";

-						logger.info(MessageFormat.format(msg, repositoryName, duration));

+					if (result.success) {

+						if (result.commitCount > 0) {

+							String msg = "Built {0} Lucene index from {1} commits in {2} msecs";

+							logger.info(MessageFormat.format(msg, repositoryName,

+									result.commitCount, duration));

+						}

 					} else {

 						String msg = "Could not build {0} Lucene index!";

 						logger.error(MessageFormat.format(msg, repositoryName));

@@ -156,11 +160,14 @@ public class LuceneExecutor implements Runnable {
 				} else {

 					// update the index with latest commits

 					long start = System.currentTimeMillis();

-					boolean success = LuceneUtils.updateIndex(repository);

+					IndexResult result = LuceneUtils.updateIndex(repository);

 					long duration = System.currentTimeMillis() - start;

-					if (success) {

-						String msg = "Updated {0} Lucene index in {1} msecs";

-						logger.info(MessageFormat.format(msg, repositoryName, duration));

+					if (result.success) {

+						if (result.commitCount > 0) {

+							String msg = "Updated {0} Lucene index with {1} commits in {2} msecs";

+							logger.info(MessageFormat.format(msg, repositoryName,

+									result.commitCount, duration));

+						}

 					} else {

 						String msg = "Could not update {0} Lucene index!";

 						logger.error(MessageFormat.format(msg, repositoryName));

diff --git a/src/com/gitblit/utils/LuceneUtils.java b/src/com/gitblit/utils/LuceneUtils.java
index d463cdf1..e8242362 100644
--- a/src/com/gitblit/utils/LuceneUtils.java
+++ b/src/com/gitblit/utils/LuceneUtils.java
@@ -52,6 +52,7 @@ import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.eclipse.jgit.treewalk.TreeWalk;

 import org.eclipse.jgit.util.FS;

 

+import com.gitblit.GitBlit;

 import com.gitblit.models.IssueModel;

 import com.gitblit.models.IssueModel.Attachment;

 import com.gitblit.models.PathModel.PathChangeModel;

@@ -121,10 +122,13 @@ public class LuceneUtils {
 	 * @return the repository name

 	 */

 	private static String getName(Repository repository) {

+		String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath();

 		if (repository.isBare()) {

-			return repository.getDirectory().getName();

+			return StringUtils.getRelativePath(rootPath, repository.getDirectory()

+					.getAbsolutePath());

 		} else {

-			return repository.getDirectory().getParentFile().getName();

+			return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile()

+					.getAbsolutePath());

 		}

 	}

 

@@ -198,11 +202,12 @@ public class LuceneUtils {
 	 * index.

 	 * 

 	 * @param repository

-	 * @return true if the indexing has succeeded

+	 * @return IndexResult

 	 */

-	public static boolean reindex(Repository repository) {

+	public static IndexResult reindex(Repository repository) {

+		IndexResult result = new IndexResult();

 		if (!LuceneUtils.deleteIndex(repository)) {

-			return false;

+			return result;

 		}

 		try {

 			String repositoryName = getName(repository);

@@ -300,6 +305,7 @@ public class LuceneUtils {
 							Index.NOT_ANALYZED));

 					doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

 					writer.addDocument(doc);

+					result.commitCount += 1;

 				}

 

 				// traverse the log and index the previous commit objects

@@ -312,6 +318,7 @@ public class LuceneUtils {
 								Index.NOT_ANALYZED));

 						doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

 						writer.addDocument(doc);

+						result.commitCount += 1;

 					}

 				}

 

@@ -335,11 +342,11 @@ public class LuceneUtils {
 			config.save();

 			resetIndexSearcher(repository);

 			writer.commit();

-			return true;

+			result.success = true;

 		} catch (Exception e) {

 			e.printStackTrace();

 		}

-		return false;

+		return result;

 	}

 

 	/**

@@ -453,9 +460,10 @@ public class LuceneUtils {
 	 * Updates a repository index incrementally from the last indexed commits.

 	 * 

 	 * @param repository

+	 * @return IndexResult

 	 */

-	public static boolean updateIndex(Repository repository) {

-		boolean success = false;

+	public static IndexResult updateIndex(Repository repository) {

+		IndexResult result = new IndexResult();

 		try {

 			FileBasedConfig config = getConfig(repository);

 			config.load();

@@ -475,13 +483,13 @@ public class LuceneUtils {
 

 			// detect branch deletion

 			// first assume all branches are deleted and then remove each

-			// existing branch from deletedBranches during indexing			

+			// existing branch from deletedBranches during indexing

 			Set<String> deletedBranches = new TreeSet<String>();

 			for (String alias : config.getNames(CONF_ALIAS)) {

 				String branch = config.getString(CONF_ALIAS, null, alias);

 				deletedBranches.add(branch);

 			}

-			

+

 			// walk through each branches

 			List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1);

 			for (RefModel branch : branches) {

@@ -490,7 +498,7 @@ public class LuceneUtils {
 				// remove this branch from the deletedBranches set

 				deletedBranches.remove(branchName);

 

-				// determine last commit				

+				// determine last commit

 				String keyName = getBranchKey(branchName);

 				String lastCommit = config.getString(CONF_BRANCH, null, keyName);

 

@@ -507,6 +515,7 @@ public class LuceneUtils {
 				Collections.reverse(revs);

 				for (RevCommit commit : revs) {

 					index(repository, branchName, commit);

+					result.commitCount += 1;

 				}

 

 				// update the config

@@ -515,7 +524,7 @@ public class LuceneUtils {
 				config.setString(CONF_BRANCH, null, keyName, branch.getObjectId().getName());

 				config.save();

 			}

-			

+

 			// the deletedBranches set will normally be empty by this point

 			// unless a branch really was deleted and no longer exists

 			if (deletedBranches.size() > 0) {

@@ -525,11 +534,11 @@ public class LuceneUtils {
 					writer.commit();

 				}

 			}

-			success = true;

+			result.success = true;

 		} catch (Throwable t) {

 			t.printStackTrace();

 		}

-		return success;

+		return result;

 	}

 

 	/**

@@ -782,4 +791,9 @@ public class LuceneUtils {
 		}

 		SEARCHERS.clear();

 	}

+

+	public static class IndexResult {

+		public boolean success;

+		public int commitCount;

+	}

 }

diff --git a/src/com/gitblit/wicket/pages/RootPage.java b/src/com/gitblit/wicket/pages/RootPage.java
index bad0140b..d81fae1b 100644
--- a/src/com/gitblit/wicket/pages/RootPage.java
+++ b/src/com/gitblit/wicket/pages/RootPage.java
@@ -101,6 +101,9 @@ public abstract class RootPage extends BasePage {
 		pages.add(new PageRegistration("gb.repositories", RepositoriesPage.class,

 				getRootPageParameters()));

 		pages.add(new PageRegistration("gb.activity", ActivityPage.class, getRootPageParameters()));

+		if (GitBlit.getBoolean(Keys.lucene.enable, false)) {

+			pages.add(new PageRegistration("gb.search", LucenePage.class));

+		}

 		if (showAdmin) {

 			pages.add(new PageRegistration("gb.users", UsersPage.class));

 		}