SONAR-16246 Show correct Project Count in Telemetry and System Info endpoint

diff --git a/server/sonar-webserver-auth/src/main/java/org/sonar/server/user/ServerUserSession.java b/server/sonar-webserver-auth/src/main/java/org/sonar/server/user/ServerUserSession.java
index b8d9dfa21b5ea786b786010c0de792726785f378..1809f5a49386b4b1e83078466fffb128e9f7cb3b 100644 (file)
--- a/server/sonar-webserver-auth/src/main/java/org/sonar/server/user/ServerUserSession.java
+++ b/server/sonar-webserver-auth/src/main/java/org/sonar/server/user/ServerUserSession.java
@@ -196,8 +196,7 @@ public class ServerUserSession extends AbstractUserSession {
 
     return portfolioHierarchyComponentUuids
       .stream()
-      .map(uuid -> hasPermission(permission, uuid))
-      .allMatch(Boolean::valueOf);
+      .allMatch(uuid -> hasPermission(permission, uuid));
   }
 
   private boolean hasPermission(String permission, String projectUuid) {

diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java
index 64808c2d0ebd8b2a787786f47ee817e1f833c535..5a5a146ebc360e09021d7590f4702663eb29f1fb 100644 (file)
--- a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java
+++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java
@@ -34,7 +34,9 @@ import java.util.stream.Stream;
 import javax.annotation.Nullable;
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
@@ -143,6 +145,8 @@ public class ProjectMeasuresIndex {
   private static final double[] COVERAGE_THRESHOLDS = {30D, 50D, 70D, 80D};
   private static final double[] SECURITY_REVIEW_RATING_THRESHOLDS = {30D, 50D, 70D, 80D};
   private static final double[] DUPLICATIONS_THRESHOLDS = {3D, 5D, 10D, 20D};
+  private static final int SCROLL_SIZE = 5000;
+  private static final TimeValue KEEP_ALIVE_SCROLL_DURATION = TimeValue.timeValueMinutes(1L);
 
   public enum Facet {
     NCLOC(new RangeMeasureFacet(NCLOC_KEY, LINES_THRESHOLDS)),
@@ -242,6 +246,12 @@ public class ProjectMeasuresIndex {
   }
 
   public ProjectMeasuresStatistics searchTelemetryStatistics() {
+    SearchRequest projectMeasuresSearchRequest = buildProjectMeasureSearchRequest();
+    SearchResponse projectMeasures = client.search(projectMeasuresSearchRequest);
+    return buildProjectMeasuresStatistics(projectMeasures);
+  }
+
+  private static SearchRequest buildProjectMeasureSearchRequest() {
     SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
       .fetchSource(false)
       .size(0);
@@ -262,18 +272,19 @@ public class ProjectMeasuresIndex {
         .minDocCount(1)
         .order(BucketOrder.count(false))
         .subAggregation(sum(FIELD_NCLOC_DISTRIBUTION_NCLOC).field(FIELD_NCLOC_DISTRIBUTION_NCLOC))));
-
     searchSourceBuilder.aggregation(AggregationBuilders.nested(NCLOC_KEY, FIELD_MEASURES)
       .subAggregation(AggregationBuilders.filter(NCLOC_KEY + "_filter", termQuery(FIELD_MEASURES_MEASURE_KEY, NCLOC_KEY))
         .subAggregation(sum(NCLOC_KEY + "_filter_sum").field(FIELD_MEASURES_MEASURE_VALUE))));
+    searchSourceBuilder.size(SCROLL_SIZE);
 
-    ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder();
-
-    SearchResponse response = client.search(EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
-      .source(searchSourceBuilder));
+    return EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType()).source(searchSourceBuilder).scroll(KEEP_ALIVE_SCROLL_DURATION);
+  }
 
+  private static ProjectMeasuresStatistics buildProjectMeasuresStatistics(SearchResponse response) {
+    ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder();
     statistics.setProjectCount(getTotalHits(response.getHits().getTotalHits()).value);
     statistics.setProjectCountByLanguage(termsToMap(response.getAggregations().get(FIELD_LANGUAGES)));
+
     Function<Terms.Bucket, Long> bucketToNcloc = bucket -> Math.round(((Sum) bucket.getAggregations().get(FIELD_NCLOC_DISTRIBUTION_NCLOC)).getValue());
     Map<String, Long> nclocByLanguage = Stream.of((Nested) response.getAggregations().get(FIELD_NCLOC_DISTRIBUTION))
       .map(nested -> (Terms) nested.getAggregations().get(nested.getName() + "_terms"))

diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java
index e4b240b3fb3b7b0aa3536c7a12e2e6bb552d00a4..f8344a01f17487c6ddcab1d997e9af767b8bd199 100644 (file)
--- a/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java
+++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java
@@ -1638,6 +1638,36 @@ public class ProjectMeasuresIndexTest {
       entry("java", 500L), entry("cs", 250L), entry("js", 50L), entry("python", 100L), entry("kotlin", 404L));
   }
 
+  @Test
+  public void search_statistics_for_large_instances() {
+    int nbProjects = 25000;
+    int javaLocByProjects = 100;
+    int jsLocByProjects = 900;
+    int csLocByProjects = 2;
+
+    ProjectMeasuresDoc[] documents = IntStream.range(0, nbProjects).mapToObj(i ->
+      newDoc("lines", 10, "coverage", 80)
+        .setLanguages(asList("java", "cs", "js"))
+        .setNclocLanguageDistributionFromMap(ImmutableMap.of("java", javaLocByProjects, "cs", csLocByProjects, "js", jsLocByProjects))).toArray(ProjectMeasuresDoc[]::new);
+
+    es.putDocuments(TYPE_PROJECT_MEASURES, documents);
+
+    ProjectMeasuresStatistics result = underTest.searchTelemetryStatistics();
+
+    assertThat(result.getProjectCount()).isEqualTo(nbProjects);
+    assertThat(result.getProjectCountByLanguage())
+      .hasSize(3)
+      .containsEntry("java", (long) nbProjects)
+      .containsEntry("cs", (long) nbProjects)
+      .containsEntry("js", (long) nbProjects);
+
+    assertThat(result.getNclocByLanguage())
+      .hasSize(3)
+      .containsEntry("java",(long) nbProjects * javaLocByProjects)
+      .containsEntry("cs",(long) nbProjects * csLocByProjects)
+      .containsEntry("js",(long) nbProjects * jsLocByProjects);
+  }
+
   @Test
   public void search_statistics_should_ignore_applications() {
     es.putDocuments(TYPE_PROJECT_MEASURES,