From 78fda912bd3c41634a52eb15234dd69abfab2829 Mon Sep 17 00:00:00 2001 From: Aurelien Poscia Date: Wed, 1 Jun 2022 17:15:53 +0200 Subject: [PATCH] SONAR-16246 Show correct Project Count in Telemetry and System Info endpoint --- .../sonar/server/user/ServerUserSession.java | 3 +- .../measure/index/ProjectMeasuresIndex.java | 21 +++++++++---- .../index/ProjectMeasuresIndexTest.java | 30 +++++++++++++++++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/server/sonar-webserver-auth/src/main/java/org/sonar/server/user/ServerUserSession.java b/server/sonar-webserver-auth/src/main/java/org/sonar/server/user/ServerUserSession.java index b8d9dfa21b5..1809f5a4938 100644 --- a/server/sonar-webserver-auth/src/main/java/org/sonar/server/user/ServerUserSession.java +++ b/server/sonar-webserver-auth/src/main/java/org/sonar/server/user/ServerUserSession.java @@ -196,8 +196,7 @@ public class ServerUserSession extends AbstractUserSession { return portfolioHierarchyComponentUuids .stream() - .map(uuid -> hasPermission(permission, uuid)) - .allMatch(Boolean::valueOf); + .allMatch(uuid -> hasPermission(permission, uuid)); } private boolean hasPermission(String permission, String projectUuid) { diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java index 64808c2d0eb..5a5a146ebc3 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java @@ -34,7 +34,9 @@ import java.util.stream.Stream; import javax.annotation.Nullable; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; @@ -143,6 +145,8 @@ public class ProjectMeasuresIndex { private static final double[] COVERAGE_THRESHOLDS = {30D, 50D, 70D, 80D}; private static final double[] SECURITY_REVIEW_RATING_THRESHOLDS = {30D, 50D, 70D, 80D}; private static final double[] DUPLICATIONS_THRESHOLDS = {3D, 5D, 10D, 20D}; + private static final int SCROLL_SIZE = 5000; + private static final TimeValue KEEP_ALIVE_SCROLL_DURATION = TimeValue.timeValueMinutes(1L); public enum Facet { NCLOC(new RangeMeasureFacet(NCLOC_KEY, LINES_THRESHOLDS)), @@ -242,6 +246,12 @@ public class ProjectMeasuresIndex { } public ProjectMeasuresStatistics searchTelemetryStatistics() { + SearchRequest projectMeasuresSearchRequest = buildProjectMeasureSearchRequest(); + SearchResponse projectMeasures = client.search(projectMeasuresSearchRequest); + return buildProjectMeasuresStatistics(projectMeasures); + } + + private static SearchRequest buildProjectMeasureSearchRequest() { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .fetchSource(false) .size(0); @@ -262,18 +272,19 @@ public class ProjectMeasuresIndex { .minDocCount(1) .order(BucketOrder.count(false)) .subAggregation(sum(FIELD_NCLOC_DISTRIBUTION_NCLOC).field(FIELD_NCLOC_DISTRIBUTION_NCLOC)))); - searchSourceBuilder.aggregation(AggregationBuilders.nested(NCLOC_KEY, FIELD_MEASURES) .subAggregation(AggregationBuilders.filter(NCLOC_KEY + "_filter", termQuery(FIELD_MEASURES_MEASURE_KEY, NCLOC_KEY)) .subAggregation(sum(NCLOC_KEY + "_filter_sum").field(FIELD_MEASURES_MEASURE_VALUE)))); + searchSourceBuilder.size(SCROLL_SIZE); - ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder(); - - SearchResponse response = client.search(EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .source(searchSourceBuilder)); + return EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType()).source(searchSourceBuilder).scroll(KEEP_ALIVE_SCROLL_DURATION); + } + private static ProjectMeasuresStatistics buildProjectMeasuresStatistics(SearchResponse response) { + ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder(); statistics.setProjectCount(getTotalHits(response.getHits().getTotalHits()).value); statistics.setProjectCountByLanguage(termsToMap(response.getAggregations().get(FIELD_LANGUAGES))); + Function bucketToNcloc = bucket -> Math.round(((Sum) bucket.getAggregations().get(FIELD_NCLOC_DISTRIBUTION_NCLOC)).getValue()); Map nclocByLanguage = Stream.of((Nested) response.getAggregations().get(FIELD_NCLOC_DISTRIBUTION)) .map(nested -> (Terms) nested.getAggregations().get(nested.getName() + "_terms")) diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java index e4b240b3fb3..f8344a01f17 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java @@ -1638,6 +1638,36 @@ public class ProjectMeasuresIndexTest { entry("java", 500L), entry("cs", 250L), entry("js", 50L), entry("python", 100L), entry("kotlin", 404L)); } + @Test + public void search_statistics_for_large_instances() { + int nbProjects = 25000; + int javaLocByProjects = 100; + int jsLocByProjects = 900; + int csLocByProjects = 2; + + ProjectMeasuresDoc[] documents = IntStream.range(0, nbProjects).mapToObj(i -> + newDoc("lines", 10, "coverage", 80) + .setLanguages(asList("java", "cs", "js")) + .setNclocLanguageDistributionFromMap(ImmutableMap.of("java", javaLocByProjects, "cs", csLocByProjects, "js", jsLocByProjects))).toArray(ProjectMeasuresDoc[]::new); + + es.putDocuments(TYPE_PROJECT_MEASURES, documents); + + ProjectMeasuresStatistics result = underTest.searchTelemetryStatistics(); + + assertThat(result.getProjectCount()).isEqualTo(nbProjects); + assertThat(result.getProjectCountByLanguage()) + .hasSize(3) + .containsEntry("java", (long) nbProjects) + .containsEntry("cs", (long) nbProjects) + .containsEntry("js", (long) nbProjects); + + assertThat(result.getNclocByLanguage()) + .hasSize(3) + .containsEntry("java",(long) nbProjects * javaLocByProjects) + .containsEntry("cs",(long) nbProjects * csLocByProjects) + .containsEntry("js",(long) nbProjects * jsLocByProjects); + } + @Test public void search_statistics_should_ignore_applications() { es.putDocuments(TYPE_PROJECT_MEASURES, -- 2.39.5