From b8a726c36d69219193c38529d456f1dd80875361 Mon Sep 17 00:00:00 2001 From: Jacek Date: Tue, 8 Dec 2020 08:49:35 +0100 Subject: [PATCH] SONAR-14224 return correct number of total issues in case index.max_result_window exceeded --- .../component/index/ComponentIndex.java | 1 + .../sonar/server/issue/index/IssueIndex.java | 3 +- .../measure/index/ProjectMeasuresIndex.java | 1 + .../index/ComponentIndexSearchTest.java | 28 +++++++++++++++++++ .../server/issue/index/IssueIndexTest.java | 20 +++++++++++++ .../index/ProjectMeasuresIndexTest.java | 11 ++++++++ 6 files changed, 63 insertions(+), 1 deletion(-) diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java index 886f81583ec..ae755e5eee5 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java @@ -83,6 +83,7 @@ public class ComponentIndex { public SearchIdResult search(ComponentQuery query, SearchOptions searchOptions) { SearchSourceBuilder source = new SearchSourceBuilder() .fetchSource(false) + .trackTotalHits(true) .from(searchOptions.getOffset()) .size(searchOptions.getLimit()); diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 0f016ac6c26..8c35b98e40f 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -360,7 +360,8 @@ public class IssueIndex { configureQuery(sourceBuilder, filterComputer); configureTopFilters(sourceBuilder, filterComputer); - sourceBuilder.fetchSource(false); + sourceBuilder.fetchSource(false) + .trackTotalHits(true); return client.search(requestBuilder); } diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java index bf81c3a02d7..635037a7b4e 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java @@ -214,6 +214,7 @@ public class ProjectMeasuresIndex { public SearchIdResult search(ProjectMeasuresQuery query, SearchOptions searchOptions) { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .fetchSource(false) + .trackTotalHits(true) .from(searchOptions.getOffset()) .size(searchOptions.getLimit()); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java index d7af68df6e3..7b288e55076 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java @@ -29,6 +29,7 @@ import org.sonar.api.resources.Qualifiers; import org.sonar.api.utils.System2; import org.sonar.db.DbTester; import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ComponentTesting; import org.sonar.db.organization.OrganizationDto; import org.sonar.server.es.EsTester; import org.sonar.server.es.SearchIdResult; @@ -40,6 +41,7 @@ import org.sonar.server.tester.UserSessionRule; import static java.util.Collections.singleton; import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT; public class ComponentIndexSearchTest { @Rule @@ -128,6 +130,18 @@ public class ComponentIndexSearchTest { assertThat(result.getUuids()).containsExactlyInAnyOrder(projects.get(3).uuid(), projects.get(4).uuid(), projects.get(5).uuid()); } + @Test + public void returns_correct_total_number_if_default_index_window_exceeded() { + userSession.logIn().setRoot(); + + index(IntStream.range(0, 12_000) + .mapToObj(i -> newDoc(ComponentTesting.newPrivateProjectDto(db.getDefaultOrganization()))) + .toArray(ComponentDoc[]::new)); + + SearchIdResult result = underTest.search(ComponentQuery.builder().build(), new SearchOptions().setPage(2, 3)); + assertThat(result.getTotal()).isEqualTo(12_000); + } + @Test public void filter_unauthorized_components() { ComponentDto unauthorizedProject = db.components().insertPrivateProject(); @@ -147,4 +161,18 @@ public class ComponentIndexSearchTest { indexer.indexAll(); Arrays.stream(components).forEach(c -> authorizationIndexerTester.allowOnlyAnyone(c)); } + + private void index(ComponentDoc... componentDocs) { + es.putDocuments(TYPE_COMPONENT.getMainType(), componentDocs); + } + + private ComponentDoc newDoc(ComponentDto componentDoc) { + return new ComponentDoc() + .setId(componentDoc.uuid()) + .setKey(componentDoc.getKey()) + .setName(componentDoc.name()) + .setProjectUuid(componentDoc.projectUuid()) + .setOrganization(componentDoc.getOrganizationUuid()) + .setQualifier(componentDoc.qualifier()); + } } diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java index 8563d65c54f..ef043c981d1 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; import org.assertj.core.groups.Tuple; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHit; @@ -130,6 +131,25 @@ public class IssueIndexTest { assertThat(result.getHits().getHits()).hasSize(SearchOptions.MAX_PAGE_SIZE); } + // SONAR-14224 + @Test + public void search_exceeding_default_index_max_window() { + ComponentDto project = newPrivateProjectDto(newOrganizationDto()); + ComponentDto file = newFileDto(project, null); + List issues = new ArrayList<>(); + for (int i = 0; i < 11_000; i++) { + String key = "I" + i; + issues.add(newDoc(key, file)); + } + indexIssues(issues.toArray(new IssueDoc[] {})); + + IssueQuery.Builder query = IssueQuery.builder(); + SearchResponse result = underTest.search(query.build(), new SearchOptions().setLimit(500)); + assertThat(result.getHits().getHits()).hasSize(SearchOptions.MAX_PAGE_SIZE); + assertThat(result.getHits().getTotalHits().value).isEqualTo(11_000L); + assertThat(result.getHits().getTotalHits().relation).isEqualTo(Relation.EQUAL_TO); + } + @Test public void authorized_issues_on_groups() { OrganizationDto org = newOrganizationDto(); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java index be7cdf6be7d..4e6ae085da2 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java @@ -513,6 +513,17 @@ public class ProjectMeasuresIndexTest { APP1, APP2, APP3); } + @Test + public void return_correct_number_of_total_if_exceeds_index_max_results() { + index(IntStream.range(0, 12_000) + .mapToObj(operand -> newDoc(ComponentTesting.newPrivateProjectDto(ORG))) + .toArray(ProjectMeasuresDoc[]::new)); + + ProjectMeasuresQuery query = new ProjectMeasuresQuery(); + SearchIdResult result = underTest.search(query, new SearchOptions()); + assertThat(result.getTotal()).isEqualTo(12_000); + } + @Test public void return_only_projects_and_applications_authorized_for_user() { indexForUser(USER1, newDoc(PROJECT1), newDoc(PROJECT2), -- 2.39.5