SONAR-8694 use only one component search request for all qualifiers

8 files changed, 148 insertions, 61 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java
index d7459b71f47..8c1e4d15b29 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java
@@ -21,6 +21,8 @@ package org.sonar.server.component.index;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -30,6 +32,13 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters;
+import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters.Bucket;
+import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsBuilder;
 import org.sonar.core.util.stream.Collectors;
 import org.sonar.server.es.BaseIndex;
 import org.sonar.server.es.EsClient;
@@ -47,6 +56,8 @@ import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COM
 
 public class ComponentIndex extends BaseIndex {
 
+  private static final String FILTERS_AGGREGATION_NAME = "filters";
+  private static final String DOCS_AGGREGATION_NAME = "docs";
   private final UserSession userSession;
 
   public ComponentIndex(EsClient client, UserSession userSession) {
@@ -54,34 +65,51 @@ public class ComponentIndex extends BaseIndex {
     this.userSession = userSession;
   }
 
-  public List<String> search(ComponentIndexQuery query) {
+  public List<ComponentsPerQualifier> search(ComponentIndexQuery query) {
     return search(query, ComponentIndexSearchFeature.values());
   }
 
   @VisibleForTesting
-  List<String> search(ComponentIndexQuery query, ComponentIndexSearchFeature... features) {
+  List<ComponentsPerQualifier> search(ComponentIndexQuery query, ComponentIndexSearchFeature... features) {
+    Collection<String> qualifiers = query.getQualifiers();
+    if (qualifiers.isEmpty()) {
+      return Collections.emptyList();
+    }
+
     SearchRequestBuilder request = getClient()
       .prepareSearch(INDEX_COMPONENTS)
       .setTypes(TYPE_COMPONENT)
-      .setFetchSource(false);
+      .setQuery(createQuery(query, features))
+      .addAggregation(createAggregation(query))
+
+      // the search hits are part of the aggregations
+      .setSize(0);
 
-    query.getLimit().ifPresent(request::setSize);
+    SearchResponse response = request.get();
+
+    return aggregationsToQualifiers(response);
+  }
 
-    request.setQuery(createQuery(query, features));
+  private static FiltersAggregationBuilder createAggregation(ComponentIndexQuery query) {
+    FiltersAggregationBuilder filtersAggregation = AggregationBuilders.filters(FILTERS_AGGREGATION_NAME)
+      .subAggregation(createSubAggregation(query));
 
-    SearchResponse searchResponse = request.get();
+    query.getQualifiers().stream()
+      .forEach(q -> filtersAggregation.filter(q, termQuery(FIELD_QUALIFIER, q)));
 
-    return Arrays.stream(searchResponse.getHits().hits())
-      .map(SearchHit::getId)
-      .collect(Collectors.toList());
+    return filtersAggregation;
+  }
+
+  private static TopHitsBuilder createSubAggregation(ComponentIndexQuery query) {
+    TopHitsBuilder sub = AggregationBuilders.topHits(DOCS_AGGREGATION_NAME);
+    query.getLimit().ifPresent(sub::setSize);
+    return sub.setFetchSource(false);
   }
 
   private QueryBuilder createQuery(ComponentIndexQuery query, ComponentIndexSearchFeature... features) {
     BoolQueryBuilder esQuery = boolQuery();
     esQuery.filter(createAuthorizationFilter());
 
-    query.getQualifier().ifPresent(q -> esQuery.filter(termQuery(FIELD_QUALIFIER, q)));
-
     BoolQueryBuilder featureQuery = boolQuery();
 
     Arrays.stream(features)
@@ -106,4 +134,24 @@ public class ComponentIndex extends BaseIndex {
     return QueryBuilders.hasParentQuery(TYPE_AUTHORIZATION,
       QueryBuilders.boolQuery().must(matchAllQuery()).filter(groupsAndUser));
   }
+
+  private static List<ComponentsPerQualifier> aggregationsToQualifiers(SearchResponse response) {
+    InternalFilters filtersAgg = response.getAggregations().get(FILTERS_AGGREGATION_NAME);
+    List<Bucket> buckets = filtersAgg.getBuckets();
+    return buckets.stream()
+      .map(ComponentIndex::bucketToQualifier)
+      .collect(Collectors.toList(buckets.size()));
+  }
+
+  private static ComponentsPerQualifier bucketToQualifier(Bucket bucket) {
+    InternalTopHits docs = bucket.getAggregations().get(DOCS_AGGREGATION_NAME);
+
+    SearchHits hitList = docs.getHits();
+    SearchHit[] hits = hitList.getHits();
+
+    List<String> componentUuids = Arrays.stream(hits).map(SearchHit::getId)
+      .collect(Collectors.toList(hits.length));
+
+    return new ComponentsPerQualifier(bucket.getKey(), componentUuids, hitList.totalHits());
+  }
 }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexQuery.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexQuery.java
index da85a3b0e41..18601bd7c7c 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexQuery.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexQuery.java
@@ -19,8 +19,9 @@
  */
 package org.sonar.server.component.index;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Optional;
-import javax.annotation.Nullable;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.util.Objects.requireNonNull;
@@ -28,7 +29,7 @@ import static java.util.Objects.requireNonNull;
 public class ComponentIndexQuery {
 
   private final String query;
-  private Optional<String> qualifier = Optional.empty();
+  private Collection<String> qualifiers = Collections.emptyList();
   private Optional<Integer> limit = Optional.empty();
 
   public ComponentIndexQuery(String query) {
@@ -37,13 +38,13 @@ public class ComponentIndexQuery {
     this.query = query;
   }
 
-  public ComponentIndexQuery setQualifier(@Nullable String qualifier) {
-    this.qualifier = Optional.ofNullable(qualifier);
+  public ComponentIndexQuery setQualifiers(Collection<String> qualifiers) {
+    this.qualifiers = Collections.unmodifiableCollection(qualifiers);
     return this;
   }
 
-  public Optional<String> getQualifier() {
-    return qualifier;
+  public Collection<String> getQualifiers() {
+    return qualifiers;
   }
 
   public String getQuery() {
diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentsPerQualifier.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentsPerQualifier.java
new file mode 100644
index 00000000000..98048e9123d
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentsPerQualifier.java
@@ -0,0 +1,47 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.component.index;
+
+import java.util.List;
+
+public class ComponentsPerQualifier {
+
+  private final String qualifier;
+  private final List<String> componentUuids;
+  private final long totalHits;
+
+  public ComponentsPerQualifier(String qualifier, List<String> componentUuids, long totalHits) {
+    this.qualifier = qualifier;
+    this.componentUuids = componentUuids;
+    this.totalHits = totalHits;
+  }
+
+  public String getQualifier() {
+    return qualifier;
+  }
+
+  public List<String> getComponentUuids() {
+    return componentUuids;
+  }
+
+  public long getTotalHits() {
+    return totalHits;
+  }
+}
diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java
index 1b6a29d0792..e2815e69409 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java
@@ -21,10 +21,9 @@ package org.sonar.server.component.ws;
 
 import com.google.common.io.Resources;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Stream;
 import org.sonar.api.resources.Qualifiers;
 import org.sonar.api.server.ws.Request;
 import org.sonar.api.server.ws.Response;
@@ -37,6 +36,7 @@ import org.sonar.db.component.ComponentDto;
 import org.sonar.db.organization.OrganizationDto;
 import org.sonar.server.component.index.ComponentIndex;
 import org.sonar.server.component.index.ComponentIndexQuery;
+import org.sonar.server.component.index.ComponentsPerQualifier;
 import org.sonarqube.ws.WsComponents.Component;
 import org.sonarqube.ws.WsComponents.SuggestionsWsResponse;
 import org.sonarqube.ws.WsComponents.SuggestionsWsResponse.Qualifier;
@@ -99,40 +99,33 @@ public class SuggestionsAction implements ComponentsWsAction {
   }
 
   private List<Qualifier> getResultsOfAllQualifiers(String query) {
-    return Arrays
-      .stream(QUALIFIERS)
-      .flatMap(qualifier -> getResultsOfQualifier(query, qualifier).map(Stream::of).orElseGet(Stream::empty))
-      .collect(Collectors.toList());
-  }
-
-  private Optional<Qualifier> getResultsOfQualifier(String query, String qualifier) {
     ComponentIndexQuery componentIndexQuery = new ComponentIndexQuery(query)
-      .setQualifier(qualifier)
+      .setQualifiers(Arrays.asList(QUALIFIERS))
       .setLimit(NUMBER_OF_RESULTS_PER_QUALIFIER);
 
-    List<String> uuids = searchInIndex(componentIndexQuery);
-    if (uuids.isEmpty()) {
-      return Optional.empty();
+    List<ComponentsPerQualifier> componentsPerQualifiers = searchInIndex(componentIndexQuery);
+
+    if (componentsPerQualifiers.isEmpty()) {
+      return Collections.emptyList();
     }
 
-    List<ComponentDto> componentDtos;
-    Map<String, String> organizationKeyByUuids;
     try (DbSession dbSession = dbClient.openSession(false)) {
-      componentDtos = dbClient.componentDao().selectByUuids(dbSession, uuids);
-      organizationKeyByUuids = getOrganizationKeys(dbSession, componentDtos);
-    }
+      return componentsPerQualifiers.stream().map(qualifier -> {
 
-    List<Component> results = componentDtos
-      .stream()
-      .map(dto -> dtoToComponent(dto, organizationKeyByUuids))
-      .collect(Collectors.toList());
+        List<ComponentDto> componentDtos = dbClient.componentDao().selectByUuids(dbSession, qualifier.getComponentUuids());
+        Map<String, String> organizationKeyByUuids = getOrganizationKeys(dbSession, componentDtos);
 
-    Qualifier q = Qualifier.newBuilder()
-      .setQ(qualifier)
-      .addAllItems(results)
-      .build();
+        List<Component> results = componentDtos
+          .stream()
+          .map(dto -> dtoToComponent(dto, organizationKeyByUuids))
+          .collect(Collectors.toList());
 
-    return Optional.of(q);
+        return Qualifier.newBuilder()
+          .setQ(qualifier.getQualifier())
+          .addAllItems(results)
+          .build();
+      }).collect(Collectors.toList());
+    }
   }
 
   private Map<String, String> getOrganizationKeys(DbSession dbSession, List<ComponentDto> componentDtos) {
@@ -143,7 +136,7 @@ public class SuggestionsAction implements ComponentsWsAction {
       .collect(Collectors.uniqueIndex(OrganizationDto::getUuid, OrganizationDto::getKey));
   }
 
-  private List<String> searchInIndex(ComponentIndexQuery componentIndexQuery) {
+  private List<ComponentsPerQualifier> searchInIndex(ComponentIndexQuery componentIndexQuery) {
     return index.search(componentIndexQuery);
   }
 
diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexCombinationTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexCombinationTest.java
index b202818ebd5..60b00dfd85e 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexCombinationTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexCombinationTest.java
@@ -24,6 +24,7 @@ import org.junit.Test;
 import org.sonar.api.resources.Qualifiers;
 import org.sonar.db.component.ComponentDto;
 
+import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class ComponentIndexCombinationTest extends ComponentIndexTest {
@@ -47,14 +48,14 @@ public class ComponentIndexCombinationTest extends ComponentIndexTest {
     ComponentDto project = indexProject("struts", "Apache Struts");
     indexFile(project, "src/main/java/StrutsManager.java", "StrutsManager.java");
 
-    assertSearchResults(new ComponentIndexQuery("struts").setQualifier(Qualifiers.PROJECT), project);
+    assertSearchResults(new ComponentIndexQuery("struts").setQualifiers(asList(Qualifiers.PROJECT)), project);
   }
 
   @Test
   public void should_limit_the_number_of_results() {
     IntStream.rangeClosed(0, 10).forEach(i -> indexProject("sonarqube" + i, "SonarQube" + i));
 
-    assertSearch(new ComponentIndexQuery("sonarqube").setLimit(5)).hasSize(5);
+    assertSearch(new ComponentIndexQuery("sonarqube").setLimit(5).setQualifiers(asList(Qualifiers.PROJECT))).hasSize(5);
   }
 
   @Test
diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java
index bc2f9746f4c..c3d16422aaa 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java
@@ -39,8 +39,12 @@ import org.sonar.server.es.EsTester;
 import org.sonar.server.permission.index.PermissionIndexerTester;
 import org.sonar.server.tester.UserSessionRule;
 
+import static java.util.Arrays.asList;
 import static java.util.Collections.emptyList;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.sonar.api.resources.Qualifiers.FILE;
+import static org.sonar.api.resources.Qualifiers.MODULE;
+import static org.sonar.api.resources.Qualifiers.PROJECT;
 
 public abstract class ComponentIndexTest {
 
@@ -101,15 +105,16 @@ public abstract class ComponentIndexTest {
   }
 
   protected AbstractListAssert<?, ? extends List<? extends String>, String> assertSearch(String query) {
-    return assertSearch(new ComponentIndexQuery(query));
+    return assertSearch(new ComponentIndexQuery(query).setQualifiers(asList(PROJECT, MODULE, FILE)));
   }
 
   protected AbstractListAssert<?, ? extends List<? extends String>, String> assertSearch(ComponentIndexQuery query) {
-    return assertThat(index.search(query, features.get()));
+    return assertThat(index.search(query, features.get()))
+      .flatExtracting(ComponentsPerQualifier::getComponentUuids);
   }
 
   protected void assertSearchResults(String query, ComponentDto... expectedComponents) {
-    assertSearchResults(new ComponentIndexQuery(query), expectedComponents);
+    assertSearchResults(new ComponentIndexQuery(query).setQualifiers(asList(PROJECT, MODULE, FILE)), expectedComponents);
   }
 
   protected void assertSearchResults(ComponentIndexQuery query, ComponentDto... expectedComponents) {
diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SuggestionsActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SuggestionsActionTest.java
index efbe3c2bb1c..7a83ddec0b0 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SuggestionsActionTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SuggestionsActionTest.java
@@ -79,6 +79,7 @@ public class SuggestionsActionTest {
 
     // assert match in qualifier "TRK"
     assertThat(response.getResultsList())
+      .filteredOn(q -> q.getItemsCount() > 0)
       .extracting(SuggestionsWsResponse.Qualifier::getQ)
       .containsExactly(Qualifiers.PROJECT);
 
diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java
index 8716be79cfa..88edbeaf2ed 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java
@@ -50,7 +50,6 @@ import org.sonar.server.component.ComponentCleanerService;
 import org.sonar.server.component.ComponentFinder;
 import org.sonar.server.component.index.ComponentIndex;
 import org.sonar.server.component.index.ComponentIndexDefinition;
-import org.sonar.server.component.index.ComponentIndexQuery;
 import org.sonar.server.component.index.ComponentIndexer;
 import org.sonar.server.es.EsTester;
 import org.sonar.server.exceptions.ForbiddenException;
@@ -166,9 +165,6 @@ public class BulkDeleteActionTest {
   public void delete_documents_indexes() throws Exception {
     IntStream.rangeClosed(1, 4).forEach(this::insertNewProjectInIndexes);
 
-    // all 4 projects should be findable
-    assertComponentIndexSearchResults("project", IntStream.rangeClosed(1, 4).mapToObj(i -> "project-uuid-" + i).toArray(String[]::new));
-
     ws.newPostRequest("api/projects", ACTION)
       .setParam(PARAM_KEYS, "project-key-1, project-key-3, project-key-4").execute();
 
@@ -178,9 +174,8 @@ public class BulkDeleteActionTest {
     assertThat(es.getIds(IssueIndexDefinition.INDEX, TYPE_AUTHORIZATION)).containsOnly(remainingProjectUuid);
     assertThat(es.getDocumentFieldValues(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, TestIndexDefinition.FIELD_PROJECT_UUID))
       .containsOnly(remainingProjectUuid);
-
-    // only the remaining project should be findable
-    assertComponentIndexSearchResults("project", remainingProjectUuid);
+    assertThat(es.getIds(ComponentIndexDefinition.INDEX_COMPONENTS, ComponentIndexDefinition.TYPE_COMPONENT)).containsOnly(remainingProjectUuid);
+    assertThat(es.getIds(ComponentIndexDefinition.INDEX_COMPONENTS, ComponentIndexDefinition.TYPE_AUTHORIZATION)).containsOnly(remainingProjectUuid);
   }
 
   @Test
@@ -219,10 +214,6 @@ public class BulkDeleteActionTest {
     ws.newPostRequest("api/projects", ACTION).setParam(PARAM_IDS, "project-uuid").execute();
   }
 
-  private void assertComponentIndexSearchResults(String query, String... expectedResultUuids) {
-    assertThat(componentIndex.search(new ComponentIndexQuery(query))).containsOnly(expectedResultUuids);
-  }
-
   private long insertNewProjectInDbAndReturnSnapshotId(int id) {
     String suffix = String.valueOf(id);
     ComponentDto project = ComponentTesting