From d040098957590010e6c94f0671cbaf1945c52098 Mon Sep 17 00:00:00 2001 From: James Moger Date: Sat, 17 Mar 2012 11:02:27 -0400 Subject: [PATCH] Implemented Lucene search result paging --- src/com/gitblit/GitBlit.java | 7 +- src/com/gitblit/LuceneExecutor.java | 33 ++++--- src/com/gitblit/models/SearchResult.java | 4 + src/com/gitblit/wicket/pages/LucenePage.html | 20 +++- src/com/gitblit/wicket/pages/LucenePage.java | 87 ++++++++++++++++- src/com/gitblit/wicket/panels/PagerPanel.html | 13 +++ src/com/gitblit/wicket/panels/PagerPanel.java | 95 +++++++++++++++++++ tests/com/gitblit/tests/IssuesTest.java | 2 +- .../com/gitblit/tests/LuceneExecutorTest.java | 22 ++--- 9 files changed, 249 insertions(+), 34 deletions(-) create mode 100644 src/com/gitblit/wicket/panels/PagerPanel.html create mode 100644 src/com/gitblit/wicket/panels/PagerPanel.java diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java index 272630c1..b2e53d67 100644 --- a/src/com/gitblit/GitBlit.java +++ b/src/com/gitblit/GitBlit.java @@ -1667,12 +1667,13 @@ public class GitBlit implements ServletContextListener { * Search the specified repositories using the Lucene query. * * @param query - * @param maximumHits + * @param page + * @param pageSize * @param repositories * @return */ - public List search(String query, int maximumHits, List repositories) { - List srs = luceneExecutor.search(query, maximumHits, repositories); + public List search(String query, int page, int pageSize, List repositories) { + List srs = luceneExecutor.search(query, page, pageSize, repositories); return srs; } diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java index 489d308d..220967be 100644 --- a/src/com/gitblit/LuceneExecutor.java +++ b/src/com/gitblit/LuceneExecutor.java @@ -940,8 +940,10 @@ public class LuceneExecutor implements Runnable { return false; } - private SearchResult createSearchResult(Document doc, float score) throws ParseException { + private SearchResult createSearchResult(Document doc, float score, int hitId, int totalHits) throws ParseException { SearchResult result = new SearchResult(); + result.hitId = hitId; + result.totalHits = totalHits; result.score = score; result.date = DateTools.stringToDate(doc.get(FIELD_DATE)); result.summary = doc.get(FIELD_SUMMARY); @@ -1017,19 +1019,21 @@ public class LuceneExecutor implements Runnable { * * @param text * if the text is null or empty, null is returned - * @param maximumHits - * the maximum number of hits to collect + * @param page + * the page number to retrieve. page is 1-indexed. + * @param pageSize + * the number of elements to return for this page * @param repositories * a list of repositories to search. if no repositories are * specified null is returned. * @return a list of SearchResults in order from highest to the lowest score * */ - public List search(String text, int maximumHits, List repositories) { + public List search(String text, int page, int pageSize, List repositories) { if (ArrayUtils.isEmpty(repositories)) { return null; } - return search(text, maximumHits, repositories.toArray(new String[0])); + return search(text, page, pageSize, repositories.toArray(new String[0])); } /** @@ -1037,15 +1041,17 @@ public class LuceneExecutor implements Runnable { * * @param text * if the text is null or empty, null is returned - * @param maximumHits - * the maximum number of hits to collect + * @param page + * the page number to retrieve. page is 1-indexed. + * @param pageSize + * the number of elements to return for this page * @param repositories * a list of repositories to search. if no repositories are * specified null is returned. * @return a list of SearchResults in order from highest to the lowest score * - */ - public List search(String text, int maximumHits, String... repositories) { + */ + public List search(String text, int page, int pageSize, String... repositories) { if (StringUtils.isEmpty(text)) { return null; } @@ -1082,14 +1088,15 @@ public class LuceneExecutor implements Runnable { searcher = new IndexSearcher(reader); } Query rewrittenQuery = searcher.rewrite(query); - TopScoreDocCollector collector = TopScoreDocCollector.create(maximumHits, true); + TopScoreDocCollector collector = TopScoreDocCollector.create(5000, true); searcher.search(rewrittenQuery, collector); - ScoreDoc[] hits = collector.topDocs().scoreDocs; + int offset = Math.max(0, (page - 1) * pageSize); + ScoreDoc[] hits = collector.topDocs(offset, pageSize).scoreDocs; + int totalHits = collector.getTotalHits(); for (int i = 0; i < hits.length; i++) { int docId = hits[i].doc; Document doc = searcher.doc(docId); - // TODO identify the source index for the doc, then eliminate FIELD_REPOSITORY - SearchResult result = createSearchResult(doc, hits[i].score); + SearchResult result = createSearchResult(doc, hits[i].score, offset + i + 1, totalHits); if (repositories.length == 1) { // single repository search result.repository = repositories[0]; diff --git a/src/com/gitblit/models/SearchResult.java b/src/com/gitblit/models/SearchResult.java index 56623075..efd1b075 100644 --- a/src/com/gitblit/models/SearchResult.java +++ b/src/com/gitblit/models/SearchResult.java @@ -15,6 +15,10 @@ import com.gitblit.Constants.SearchObjectType; public class SearchResult implements Serializable { private static final long serialVersionUID = 1L; + + public int hitId; + + public int totalHits; public float score; diff --git a/src/com/gitblit/wicket/pages/LucenePage.html b/src/com/gitblit/wicket/pages/LucenePage.html index e6a9b932..75f63364 100644 --- a/src/com/gitblit/wicket/pages/LucenePage.html +++ b/src/com/gitblit/wicket/pages/LucenePage.html @@ -48,8 +48,18 @@ -
-
+ +
+ +
+

+
+ +
+
+ +
+
@@ -58,7 +68,11 @@ :
-
+ + +
+ +
\ No newline at end of file diff --git a/src/com/gitblit/wicket/pages/LucenePage.java b/src/com/gitblit/wicket/pages/LucenePage.java index 099471a8..10de0bf5 100644 --- a/src/com/gitblit/wicket/pages/LucenePage.java +++ b/src/com/gitblit/wicket/pages/LucenePage.java @@ -15,6 +15,7 @@ */ package com.gitblit.wicket.pages; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; @@ -31,6 +32,7 @@ import org.eclipse.jgit.lib.Constants; import com.gitblit.Constants.SearchType; import com.gitblit.GitBlit; +import com.gitblit.Keys; import com.gitblit.models.RepositoryModel; import com.gitblit.models.SearchResult; import com.gitblit.models.UserModel; @@ -40,6 +42,7 @@ import com.gitblit.wicket.GitBlitWebSession; import com.gitblit.wicket.StringChoiceRenderer; import com.gitblit.wicket.WicketUtils; import com.gitblit.wicket.panels.LinkPanel; +import com.gitblit.wicket.panels.PagerPanel; public class LucenePage extends RootPage { @@ -59,12 +62,16 @@ public class LucenePage extends RootPage { // default values ArrayList repositories = new ArrayList(); String query = ""; + int page = 1; + int pageSize = GitBlit.getInteger(Keys.web.itemsPerPage, 50); if (params != null) { String repository = WicketUtils.getRepositoryName(params); if (!StringUtils.isEmpty(repository)) { repositories.add(repository); } + + page = WicketUtils.getPage(params); if (params.containsKey("repositories")) { String value = params.getString("repositories", ""); @@ -144,7 +151,18 @@ public class LucenePage extends RootPage { // execute search final List results = new ArrayList(); if (!ArrayUtils.isEmpty(searchRepositories) && !StringUtils.isEmpty(query)) { - results.addAll(GitBlit.self().search(query, 100, searchRepositories)); + results.addAll(GitBlit.self().search(query, page, pageSize, searchRepositories)); + } + + // results header + if (results.size() == 0) { + add(new Label("resultsHeader").setVisible(false)); + add(new Label("resultsCount").setVisible(false)); + } else { + add(new Label("resultsHeader", query).setRenderBodyOnly(true)); + add(new Label("resultsCount", MessageFormat.format("results {0} - {1} ({2} hits)", + results.get(0).hitId, results.get(results.size() - 1).hitId, results.get(0).totalHits)). + setRenderBodyOnly(true)); } // search results view @@ -178,11 +196,74 @@ public class LucenePage extends RootPage { } item.add(new Label("fragment", sr.fragment).setEscapeModelStrings(false).setVisible(!StringUtils.isEmpty(sr.fragment))); item.add(new LinkPanel("repository", null, sr.repository, SummaryPage.class, WicketUtils.newRepositoryParameter(sr.repository))); - item.add(new LinkPanel("branch", "branch", StringUtils.getRelativePath(Constants.R_HEADS, sr.branch), LogPage.class, WicketUtils.newObjectParameter(sr.repository, sr.branch))); + if (StringUtils.isEmpty(sr.branch)) { + item.add(new Label("branch", "null")); + } else { + item.add(new LinkPanel("branch", "branch", StringUtils.getRelativePath(Constants.R_HEADS, sr.branch), LogPage.class, WicketUtils.newObjectParameter(sr.repository, sr.branch))); + } item.add(new Label("author", sr.author)); item.add(WicketUtils.createDatestampLabel("date", sr.date, getTimeZone())); } }; add(resultsView.setVisible(results.size() > 0)); - } + + PageParameters pagerParams = new PageParameters(); + pagerParams.put("repositories", StringUtils.flattenStrings(repositoriesModel.getObject())); + pagerParams.put("query", queryModel.getObject()); + + int totalPages = 0; + if (results.size() > 0) { + totalPages = (results.get(0).totalHits / pageSize) + (results.get(0).totalHits % pageSize > 0 ? 1 : 0); + } + + add(new PagerPanel("topPager", page, totalPages, LucenePage.class, pagerParams)); + add(new PagerPanel("bottomPager", page, totalPages, LucenePage.class, pagerParams)); + } + +// private String buildPager(int currentPage, int count, int total) { +// int pages = (total / count) + (total % count == 0 ? 0 : 1); +// +// // pages are 1-indexed +// // previous page link +// if (currentPage <= 1) { +// sb.append(MessageFormat.format(li, "disabled", "#", "←")); +// } else { +// List parameters = new ArrayList(); +// if (!StringUtils.isEmpty(penString)) { +// parameters.add(penString); +// } +// parameters.add(MessageFormat.format(pg, currentPage - 1)); +// sb.append(MessageFormat.format(li, "", StringUtils.flattenStrings(parameters, "&"), "←")); +// } +// +// // page links in middle +// int minpage = Math.max(1, currentPage - Math.min(2, 2)); +// int maxpage = Math.min(pages, minpage + 4); +// for (int i = minpage; i <= maxpage; i++) { +// String cssClass = ""; +// if (i == currentPage) { +// cssClass = "active"; +// } +// List parameters = new ArrayList(); +// if (!StringUtils.isEmpty(penString)) { +// parameters.add(penString); +// } +// parameters.add(MessageFormat.format(pg, i)); +// sb.append(MessageFormat.format(li, cssClass, StringUtils.flattenStrings(parameters, "&"), i)); +// } +// +// // next page link +// if (currentPage == pages) { +// sb.append(MessageFormat.format(li, "disabled", "#", "→")); +// } else { +// List parameters = new ArrayList(); +// if (!StringUtils.isEmpty(penString)) { +// parameters.add(penString); +// } +// parameters.add(MessageFormat.format(pg, currentPage + 1)); +// sb.append(MessageFormat.format(li, "", StringUtils.flattenStrings(parameters, "&"), "→")); +// } +// return sb.toString(); +// } + } diff --git a/src/com/gitblit/wicket/panels/PagerPanel.html b/src/com/gitblit/wicket/panels/PagerPanel.html new file mode 100644 index 00000000..099001db --- /dev/null +++ b/src/com/gitblit/wicket/panels/PagerPanel.html @@ -0,0 +1,13 @@ + + + + + + \ No newline at end of file diff --git a/src/com/gitblit/wicket/panels/PagerPanel.java b/src/com/gitblit/wicket/panels/PagerPanel.java new file mode 100644 index 00000000..a5dbb9ef --- /dev/null +++ b/src/com/gitblit/wicket/panels/PagerPanel.java @@ -0,0 +1,95 @@ +/* + * Copyright 2012 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.wicket.panels; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.apache.wicket.PageParameters; +import org.apache.wicket.markup.html.panel.Panel; +import org.apache.wicket.markup.repeater.Item; +import org.apache.wicket.markup.repeater.data.DataView; +import org.apache.wicket.markup.repeater.data.ListDataProvider; + +import com.gitblit.wicket.WicketUtils; +import com.gitblit.wicket.pages.BasePage; + +public class PagerPanel extends Panel { + + private static final long serialVersionUID = 1L; + + public PagerPanel(String wicketId, final int currentPage, final int totalPages, + final Class pageClass, final PageParameters baseParams) { + super(wicketId); + List pages = new ArrayList(); + int[] deltas; + if (currentPage == 1) { + // [1], 2, 3, 4, 5 + deltas = new int[] { 0, 1, 2, 3, 4 }; + } else if (currentPage == 2) { + // 1, [2], 3, 4, 5 + deltas = new int[] { -1, 0, 1, 2, 3 }; + } else { + // 1, 2, [3], 4, 5 + deltas = new int[] { -2, -1, 0, 1, 2 }; + } + + if (totalPages > 0) { + pages.add(new PageObject("\u2190", currentPage - 1)); + } + for (int delta : deltas) { + int page = currentPage + delta; + if (page > 0 && page <= totalPages) { + pages.add(new PageObject("" + page, page)); + } + } + if (totalPages > 0) { + pages.add(new PageObject("\u2192", currentPage + 1)); + } + + ListDataProvider pagesProvider = new ListDataProvider(pages); + final DataView pagesView = new DataView("page", pagesProvider) { + private static final long serialVersionUID = 1L; + + public void populateItem(final Item item) { + PageObject pageItem = item.getModelObject(); + PageParameters pageParams = new PageParameters(baseParams); + pageParams.put("pg", pageItem.page); + LinkPanel link = new LinkPanel("pageLink", null, pageItem.text, pageClass, pageParams); + link.setRenderBodyOnly(true); + item.add(link); + if (pageItem.page == currentPage || pageItem.page < 1 || pageItem.page > totalPages) { + WicketUtils.setCssClass(item, "disabled"); + } + } + }; + add(pagesView); + } + + private class PageObject implements Serializable { + + private static final long serialVersionUID = 1L; + + String text; + int page; + + PageObject(String text, int page) { + this.text = text; + this.page = page; + } + } +} diff --git a/tests/com/gitblit/tests/IssuesTest.java b/tests/com/gitblit/tests/IssuesTest.java index 9133f9b1..e329f667 100644 --- a/tests/com/gitblit/tests/IssuesTest.java +++ b/tests/com/gitblit/tests/IssuesTest.java @@ -134,7 +134,7 @@ public class IssuesTest { for (IssueModel anIssue : allIssues) { lucene.index(name, anIssue); } - List hits = lucene.search("working", 10, name); + List hits = lucene.search("working", 1, 10, name); assertTrue(hits.size() > 0); // reindex an issue diff --git a/tests/com/gitblit/tests/LuceneExecutorTest.java b/tests/com/gitblit/tests/LuceneExecutorTest.java index 7a171dbc..d221744c 100644 --- a/tests/com/gitblit/tests/LuceneExecutorTest.java +++ b/tests/com/gitblit/tests/LuceneExecutorTest.java @@ -66,9 +66,9 @@ public class LuceneExecutorTest { lucene.reindex(model, repository); repository.close(); - SearchResult result = lucene.search("type:blob AND path:bit.bit", 1, model.name).get(0); + SearchResult result = lucene.search("type:blob AND path:bit.bit", 1, 1, model.name).get(0); assertEquals("Mike Donaghy", result.author); - result = lucene.search("type:blob AND path:clipper.prg", 1, model.name).get(0); + result = lucene.search("type:blob AND path:clipper.prg", 1, 1, model.name).get(0); assertEquals("tinogomes", result.author); // reindex theoretical physics @@ -95,18 +95,18 @@ public class LuceneExecutorTest { RepositoryModel model = newRepositoryModel(repository); repository.close(); - List results = lucene.search("ada", 10, model.name); + List results = lucene.search("ada", 1, 10, model.name); assertEquals(2, results.size()); for (SearchResult res : results) { assertEquals("refs/heads/master", res.branch); } // author test - results = lucene.search("author: tinogomes AND type:commit", 10, model.name); + results = lucene.search("author: tinogomes AND type:commit", 1, 10, model.name); assertEquals(2, results.size()); // blob test - results = lucene.search("type: blob AND \"import std.stdio\"", 10, model.name); + results = lucene.search("type: blob AND \"import std.stdio\"", 1, 10, model.name); assertEquals(1, results.size()); assertEquals("d.D", results.get(0).path); @@ -115,20 +115,20 @@ public class LuceneExecutorTest { model = newRepositoryModel(repository); repository.close(); - results = lucene.search("\"add the .nojekyll file\"", 10, model.name); + results = lucene.search("\"add the .nojekyll file\"", 1, 10, model.name); assertEquals(1, results.size()); assertEquals("Ondrej Certik", results.get(0).author); assertEquals("2648c0c98f2101180715b4d432fc58d0e21a51d7", results.get(0).commitId); assertEquals("refs/heads/gh-pages", results.get(0).branch); - results = lucene.search("type:blob AND \"src/intro.rst\"", 10, model.name); + results = lucene.search("type:blob AND \"src/intro.rst\"", 1, 10, model.name); assertEquals(4, results.size()); // hash id tests - results = lucene.search("commit:57c4f26f157ece24b02f4f10f5f68db1d2ce7ff5", 10, model.name); + results = lucene.search("commit:57c4f26f157ece24b02f4f10f5f68db1d2ce7ff5", 1, 10, model.name); assertEquals(1, results.size()); - results = lucene.search("commit:57c4f26f157*", 10, model.name); + results = lucene.search("commit:57c4f26f157*", 1, 10, model.name); assertEquals(1, results.size()); // annotated tag test @@ -136,7 +136,7 @@ public class LuceneExecutorTest { model = newRepositoryModel(repository); repository.close(); - results = lucene.search("I663208919f297836a9c16bf458e4a43ffaca4c12", 10, model.name); + results = lucene.search("I663208919f297836a9c16bf458e4a43ffaca4c12", 1, 10, model.name); assertEquals(1, results.size()); assertEquals("[v1.3.0.201202151440-r]", results.get(0).tags.toString()); @@ -155,7 +155,7 @@ public class LuceneExecutorTest { list.add(newRepositoryModel(repository).name); repository.close(); - List results = lucene.search("test", 10, list); + List results = lucene.search("test", 1, 10, list); lucene.close(); assertEquals(10, results.size()); } -- 2.39.5