Draft integration of Lucene search mechanism and web ui

diff --git a/distrib/gitblit.properties b/distrib/gitblit.properties
index 2ac6598febdbe84c601f9cda790cf95a33aed111..ede5f5964af8f0906511a1ce3c97964f566f07fb 100644 (file)
--- a/distrib/gitblit.properties
+++ b/distrib/gitblit.properties
@@ -91,6 +91,26 @@ groovy.preReceiveScripts =
 # SINCE 0.8.0\r
 groovy.postReceiveScripts =\r
 \r
+# If true, a Lucene index will be generated and maintained for each repository.\r
+# Lucene search replaces brute-force Git repository traversal.\r
+#\r
+# SINCE 0.9.0\r
+# RESTART REQUIRED\r
+lucene.enable  = false\r
+\r
+# If *lucene.pollingMode* = true, Gitblit will periodically check all repositories\r
+# for branch updates.\r
+# If *lucene.pollingMode* = false, repositories will only be indexed on pushes\r
+# to Gitblit.\r
+#\r
+# Regardless of this setting, Gitblit will check all repositories for branch\r
+# updates 1 minute after startup. Indexes will automatically be built for any\r
+# repository that is missing its index  or if an index version change is detected.\r
+#\r
+# SINCE 0.9.0\r
+# RESTART REQUIRED\r
+lucene.pollingMode = false\r
+\r
 #\r
 # Authentication Settings\r
 #\r

diff --git a/resources/gitblit.css b/resources/gitblit.css
index 6ad45fb2fa987c8460056fb0b8f0191bb4858fdf..bfa1c20213c1f1e3b903dbd02c89b2b7f409a8b0 100644 (file)
--- a/resources/gitblit.css
+++ b/resources/gitblit.css
@@ -266,6 +266,18 @@ img.gravatar {
     padding: 2px;\r
 }\r
 \r
+div.searchResult {\r
+       padding:5px;\r
+}\r
+\r
+div.searchResult .summary {\r
+       font-weight: bold;\r
+}\r
+\r
+div.searchResult .branch {\r
+       color: #008000;\r
+}\r
+\r
 div.header, div.commitHeader, table.repositories th {\r
        background-color:#e0e0e0;\r
        background-repeat:repeat-x;\r

diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java
index a7bfad4f3d1f9ec2f4340e3f9e4afa1d7eb410ba..f6691dc6d60a0ed8f5c5cd2013402b5eabd016b2 100644 (file)
--- a/src/com/gitblit/GitBlit.java
+++ b/src/com/gitblit/GitBlit.java
@@ -1824,8 +1824,8 @@ public class GitBlit implements ServletContextListener {
                }\r
                luceneExecutor = new LuceneExecutor(settings);\r
                if (luceneExecutor.isReady()) {\r
-                       logger.info("Lucene executor is scheduled to process the repository queue every 10 minutes.");\r
-                       scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 10, TimeUnit.MINUTES);\r
+                       logger.info("Lucene executor is scheduled to process the repository queue every 2 minutes.");\r
+                       scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 2, TimeUnit.MINUTES);\r
                } else {\r
                        logger.warn("Lucene executor is disabled.");\r
                }\r

diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java
index e4624efc147fd9f79dd0e226113602a62105f047..b4e5134aa1dd1d5db0ea47a398da2a430db10c52 100644 (file)
--- a/src/com/gitblit/LuceneExecutor.java
+++ b/src/com/gitblit/LuceneExecutor.java
@@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory;
 import com.gitblit.models.RepositoryModel;\r
 import com.gitblit.utils.JGitUtils;\r
 import com.gitblit.utils.LuceneUtils;\r
+import com.gitblit.utils.LuceneUtils.IndexResult;\r
 \r
 /**\r
  * The Lucene executor handles indexing repositories synchronously and\r
@@ -53,8 +54,8 @@ public class LuceneExecutor implements Runnable {
 \r
        public LuceneExecutor(IStoredSettings settings) {\r
                this.settings = settings;\r
-               this.isLuceneEnabled = settings.getBoolean("lucene.enableLucene", false);\r
-               this.isPollingMode = settings.getBoolean("lucene.pollingMode", false);\r
+               this.isLuceneEnabled = settings.getBoolean(Keys.lucene.enable, false);\r
+               this.isPollingMode = settings.getBoolean(Keys.lucene.pollingMode, false);\r
        }\r
 \r
        /**\r
@@ -144,11 +145,14 @@ public class LuceneExecutor implements Runnable {
                                if (LuceneUtils.shouldReindex(repository)) {\r
                                        // (re)build the entire index\r
                                        long start = System.currentTimeMillis();\r
-                                       boolean success = LuceneUtils.reindex(repository);\r
+                                       IndexResult result = LuceneUtils.reindex(repository);\r
                                        long duration = System.currentTimeMillis() - start;\r
-                                       if (success) {\r
-                                               String msg = "Built {0} Lucene index in {1} msecs";\r
-                                               logger.info(MessageFormat.format(msg, repositoryName, duration));\r
+                                       if (result.success) {\r
+                                               if (result.commitCount > 0) {\r
+                                                       String msg = "Built {0} Lucene index from {1} commits in {2} msecs";\r
+                                                       logger.info(MessageFormat.format(msg, repositoryName,\r
+                                                                       result.commitCount, duration));\r
+                                               }\r
                                        } else {\r
                                                String msg = "Could not build {0} Lucene index!";\r
                                                logger.error(MessageFormat.format(msg, repositoryName));\r
@@ -156,11 +160,14 @@ public class LuceneExecutor implements Runnable {
                                } else {\r
                                        // update the index with latest commits\r
                                        long start = System.currentTimeMillis();\r
-                                       boolean success = LuceneUtils.updateIndex(repository);\r
+                                       IndexResult result = LuceneUtils.updateIndex(repository);\r
                                        long duration = System.currentTimeMillis() - start;\r
-                                       if (success) {\r
-                                               String msg = "Updated {0} Lucene index in {1} msecs";\r
-                                               logger.info(MessageFormat.format(msg, repositoryName, duration));\r
+                                       if (result.success) {\r
+                                               if (result.commitCount > 0) {\r
+                                                       String msg = "Updated {0} Lucene index with {1} commits in {2} msecs";\r
+                                                       logger.info(MessageFormat.format(msg, repositoryName,\r
+                                                                       result.commitCount, duration));\r
+                                               }\r
                                        } else {\r
                                                String msg = "Could not update {0} Lucene index!";\r
                                                logger.error(MessageFormat.format(msg, repositoryName));\r

diff --git a/src/com/gitblit/utils/LuceneUtils.java b/src/com/gitblit/utils/LuceneUtils.java
index d463cdf1fdaffd64b20600f30ae000f1a55ff2cd..e824236264e261ed112c24047971abd517a7430c 100644 (file)
--- a/src/com/gitblit/utils/LuceneUtils.java
+++ b/src/com/gitblit/utils/LuceneUtils.java
@@ -52,6 +52,7 @@ import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.eclipse.jgit.treewalk.TreeWalk;\r
 import org.eclipse.jgit.util.FS;\r
 \r
+import com.gitblit.GitBlit;\r
 import com.gitblit.models.IssueModel;\r
 import com.gitblit.models.IssueModel.Attachment;\r
 import com.gitblit.models.PathModel.PathChangeModel;\r
@@ -121,10 +122,13 @@ public class LuceneUtils {
         * @return the repository name\r
         */\r
        private static String getName(Repository repository) {\r
+               String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath();\r
                if (repository.isBare()) {\r
-                       return repository.getDirectory().getName();\r
+                       return StringUtils.getRelativePath(rootPath, repository.getDirectory()\r
+                                       .getAbsolutePath());\r
                } else {\r
-                       return repository.getDirectory().getParentFile().getName();\r
+                       return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile()\r
+                                       .getAbsolutePath());\r
                }\r
        }\r
 \r
@@ -198,11 +202,12 @@ public class LuceneUtils {
         * index.\r
         * \r
         * @param repository\r
-        * @return true if the indexing has succeeded\r
+        * @return IndexResult\r
         */\r
-       public static boolean reindex(Repository repository) {\r
+       public static IndexResult reindex(Repository repository) {\r
+               IndexResult result = new IndexResult();\r
                if (!LuceneUtils.deleteIndex(repository)) {\r
-                       return false;\r
+                       return result;\r
                }\r
                try {\r
                        String repositoryName = getName(repository);\r
@@ -300,6 +305,7 @@ public class LuceneUtils {
                                                        Index.NOT_ANALYZED));\r
                                        doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));\r
                                        writer.addDocument(doc);\r
+                                       result.commitCount += 1;\r
                                }\r
 \r
                                // traverse the log and index the previous commit objects\r
@@ -312,6 +318,7 @@ public class LuceneUtils {
                                                                Index.NOT_ANALYZED));\r
                                                doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));\r
                                                writer.addDocument(doc);\r
+                                               result.commitCount += 1;\r
                                        }\r
                                }\r
 \r
@@ -335,11 +342,11 @@ public class LuceneUtils {
                        config.save();\r
                        resetIndexSearcher(repository);\r
                        writer.commit();\r
-                       return true;\r
+                       result.success = true;\r
                } catch (Exception e) {\r
                        e.printStackTrace();\r
                }\r
-               return false;\r
+               return result;\r
        }\r
 \r
        /**\r
@@ -453,9 +460,10 @@ public class LuceneUtils {
         * Updates a repository index incrementally from the last indexed commits.\r
         * \r
         * @param repository\r
+        * @return IndexResult\r
         */\r
-       public static boolean updateIndex(Repository repository) {\r
-               boolean success = false;\r
+       public static IndexResult updateIndex(Repository repository) {\r
+               IndexResult result = new IndexResult();\r
                try {\r
                        FileBasedConfig config = getConfig(repository);\r
                        config.load();\r
@@ -475,13 +483,13 @@ public class LuceneUtils {
 \r
                        // detect branch deletion\r
                        // first assume all branches are deleted and then remove each\r
-                       // existing branch from deletedBranches during indexing                 \r
+                       // existing branch from deletedBranches during indexing\r
                        Set<String> deletedBranches = new TreeSet<String>();\r
                        for (String alias : config.getNames(CONF_ALIAS)) {\r
                                String branch = config.getString(CONF_ALIAS, null, alias);\r
                                deletedBranches.add(branch);\r
                        }\r
-                       \r
+\r
                        // walk through each branches\r
                        List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1);\r
                        for (RefModel branch : branches) {\r
@@ -490,7 +498,7 @@ public class LuceneUtils {
                                // remove this branch from the deletedBranches set\r
                                deletedBranches.remove(branchName);\r
 \r
-                               // determine last commit                                \r
+                               // determine last commit\r
                                String keyName = getBranchKey(branchName);\r
                                String lastCommit = config.getString(CONF_BRANCH, null, keyName);\r
 \r
@@ -507,6 +515,7 @@ public class LuceneUtils {
                                Collections.reverse(revs);\r
                                for (RevCommit commit : revs) {\r
                                        index(repository, branchName, commit);\r
+                                       result.commitCount += 1;\r
                                }\r
 \r
                                // update the config\r
@@ -515,7 +524,7 @@ public class LuceneUtils {
                                config.setString(CONF_BRANCH, null, keyName, branch.getObjectId().getName());\r
                                config.save();\r
                        }\r
-                       \r
+\r
                        // the deletedBranches set will normally be empty by this point\r
                        // unless a branch really was deleted and no longer exists\r
                        if (deletedBranches.size() > 0) {\r
@@ -525,11 +534,11 @@ public class LuceneUtils {
                                        writer.commit();\r
                                }\r
                        }\r
-                       success = true;\r
+                       result.success = true;\r
                } catch (Throwable t) {\r
                        t.printStackTrace();\r
                }\r
-               return success;\r
+               return result;\r
        }\r
 \r
        /**\r
@@ -782,4 +791,9 @@ public class LuceneUtils {
                }\r
                SEARCHERS.clear();\r
        }\r
+\r
+       public static class IndexResult {\r
+               public boolean success;\r
+               public int commitCount;\r
+       }\r
 }\r

diff --git a/src/com/gitblit/wicket/pages/RootPage.java b/src/com/gitblit/wicket/pages/RootPage.java
index bad0140b7b5a452b4cbaa5428653859fc4999086..d81fae1bb59c49a901fe7847f79b8ef043a41b11 100644 (file)
--- a/src/com/gitblit/wicket/pages/RootPage.java
+++ b/src/com/gitblit/wicket/pages/RootPage.java
@@ -101,6 +101,9 @@ public abstract class RootPage extends BasePage {
                pages.add(new PageRegistration("gb.repositories", RepositoriesPage.class,\r
                                getRootPageParameters()));\r
                pages.add(new PageRegistration("gb.activity", ActivityPage.class, getRootPageParameters()));\r
+               if (GitBlit.getBoolean(Keys.lucene.enable, false)) {\r
+                       pages.add(new PageRegistration("gb.search", LucenePage.class));\r
+               }\r
                if (showAdmin) {\r
                        pages.add(new PageRegistration("gb.users", UsersPage.class));\r
                }\r