SONAR-9721 Aggregate ncloc by language in the project measures ES index

diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java
index 473a44ecdc878921c6192138b7f2661d0e1e8563..1015e27b289040cb6853797cd945a0b4bffab924 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java
@@ -27,6 +27,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
@@ -90,6 +91,7 @@ import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIEL
 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_LANGUAGES;
 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES;
 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NAME;
+import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_LANGUAGE_DISTRIBUTION;
 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ORGANIZATION_UUID;
 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS;
 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS;
@@ -125,6 +127,8 @@ public class ProjectMeasuresIndex {
 
   private static final String FIELD_MEASURES_KEY = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_KEY;
   private static final String FIELD_MEASURES_VALUE = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_VALUE;
+  private static final String FIELD_DISTRIB_LANGUAGE = FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "." + ProjectMeasuresIndexDefinition.FIELD_DISTRIB_LANGUAGE;
+  private static final String FIELD_DISTRIB_NCLOC = FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "." + ProjectMeasuresIndexDefinition.FIELD_DISTRIB_NCLOC;
 
   private static final Map<String, FacetSetter> FACET_FACTORIES = ImmutableMap.<String, FacetSetter>builder()
     .put(NCLOC_KEY, (esSearch, query, facetBuilder) -> addRangeFacet(esSearch, NCLOC_KEY, facetBuilder, LINES_THRESHOLDS))
@@ -184,6 +188,14 @@ public class ProjectMeasuresIndex {
       .size(MAX_PAGE_SIZE)
       .minDocCount(1)
       .order(Terms.Order.count(false)));
+    request.addAggregation(AggregationBuilders.nested(FIELD_NCLOC_LANGUAGE_DISTRIBUTION, FIELD_NCLOC_LANGUAGE_DISTRIBUTION)
+      .subAggregation(AggregationBuilders.terms(FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "_terms")
+        .field(FIELD_DISTRIB_LANGUAGE)
+        .size(MAX_PAGE_SIZE)
+        .minDocCount(1)
+        .order(Terms.Order.count(false))
+        .subAggregation(sum(FIELD_DISTRIB_NCLOC).field(FIELD_DISTRIB_NCLOC))));
+
     Stream.of(LINES_KEY, NCLOC_KEY)
       .forEach(metric -> request.addAggregation(AggregationBuilders.nested(metric, FIELD_MEASURES)
         .subAggregation(AggregationBuilders.filter(metric + "_filter", termQuery(FIELD_MEASURES_KEY, metric))
@@ -202,7 +214,13 @@ public class ProjectMeasuresIndex {
         long value = Math.round(sum.getValue());
         statistics.setSum(metric, value);
       });
-    statistics.setProjectLanguageDistribution(termsToMap(response.getAggregations().get(FIELD_LANGUAGES)));
+    statistics.setProjectCountByLanguage(termsToMap(response.getAggregations().get(FIELD_LANGUAGES)));
+    Function<Terms.Bucket, Long> bucketToNcloc = bucket -> Math.round(((Sum) bucket.getAggregations().get(FIELD_DISTRIB_NCLOC)).getValue());
+    Map<String, Long> nclocByLanguage = Stream.of((Nested) response.getAggregations().get(FIELD_NCLOC_LANGUAGE_DISTRIBUTION))
+      .map(nested -> (Terms) nested.getAggregations().get(nested.getName() + "_terms"))
+      .flatMap(terms -> terms.getBuckets().stream())
+      .collect(MoreCollectors.uniqueIndex(Bucket::getKeyAsString, bucketToNcloc));
+    statistics.setNclocByLanguage(nclocByLanguage);
 
     return statistics.build();
   }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresStatistics.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresStatistics.java
index 8d2e134b97a0d2b13d19977ef6890e11a7436b9e..c847796ee1465ef20c1ef3cc9148f542a1db0945 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresStatistics.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresStatistics.java
@@ -30,13 +30,15 @@ public class ProjectMeasuresStatistics {
   private final long projectCount;
   private final long lines;
   private final long ncloc;
-  private final Map<String, Long> projectLanguageDistribution;
+  private final Map<String, Long> projectCountByLanguage;
+  private final Map<String, Long> nclocByLanguage;
 
   private ProjectMeasuresStatistics(Builder builder) {
     projectCount = builder.projectCount;
     lines = builder.lines;
     ncloc = builder.ncloc;
-    projectLanguageDistribution = builder.projectLanguageDistribution;
+    projectCountByLanguage = builder.projectCountByLanguage;
+    nclocByLanguage = builder.nclocByLanguage;
   }
 
   public long getProjectCount() {
@@ -51,8 +53,12 @@ public class ProjectMeasuresStatistics {
     return ncloc;
   }
 
-  public Map<String, Long> getProjectLanguageDistribution() {
-    return projectLanguageDistribution;
+  public Map<String, Long> getProjectCountByLanguage() {
+    return projectCountByLanguage;
+  }
+
+  public Map<String, Long> getNclocByLanguage() {
+    return nclocByLanguage;
   }
 
   public static Builder builder() {
@@ -60,10 +66,11 @@ public class ProjectMeasuresStatistics {
   }
 
   public static class Builder {
-    private Map<String, Long> projectLanguageDistribution;
     private Long projectCount;
     private Long lines;
     private Long ncloc;
+    private Map<String, Long> projectCountByLanguage;
+    private Map<String, Long> nclocByLanguage;
 
     private Builder() {
       // enforce static factory method
@@ -88,15 +95,21 @@ public class ProjectMeasuresStatistics {
       return this;
     }
 
-    public void setProjectLanguageDistribution(Map<String, Long> projectLanguageDistribution) {
-      this.projectLanguageDistribution = projectLanguageDistribution;
+    public void setProjectCountByLanguage(Map<String, Long> projectCountByLanguage) {
+      this.projectCountByLanguage = projectCountByLanguage;
+    }
+
+    public Builder setNclocByLanguage(Map<String, Long> nclocByLanguage) {
+      this.nclocByLanguage = nclocByLanguage;
+      return this;
     }
 
     public ProjectMeasuresStatistics build() {
       requireNonNull(projectCount);
       requireNonNull(lines);
       requireNonNull(ncloc);
-      requireNonNull(projectLanguageDistribution);
+      requireNonNull(projectCountByLanguage);
+      requireNonNull(nclocByLanguage);
       return new ProjectMeasuresStatistics(this);
     }
   }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java
index e41d3258bef3db51fc6d695ebd816e5f0c40cb24..3b907b2020eceb225aa8a9f710b00f9b9b11e638 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java
@@ -165,7 +165,7 @@ public class TelemetryDaemon implements Startable {
       writer.prop(LINES_KEY, statistics.getLines());
       writer.prop(NCLOC_KEY, statistics.getNcloc());
       writer.name("projectLanguageDistribution");
-      writer.valueObject(statistics.getProjectLanguageDistribution());
+      writer.valueObject(statistics.getProjectCountByLanguage());
       writer.endObject();
     }
     telemetryClient.upload(json.toString());

diff --git a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java
index b8691c70cbf9da71ad08a692ee8b464ae73cabc6..c63fdb68e4c96deca002a53af2cf0ec33c7f0aa4 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java
@@ -1373,16 +1373,22 @@ public class ProjectMeasuresIndexTest {
   @Test
   public void search_statistics() {
     es.putDocuments(INDEX_TYPE_PROJECT_MEASURES,
-      newDoc("lines", 10, "ncloc", 20, "coverage", 80).setLanguages(Arrays.asList("java", "cs", "js")),
-      newDoc("lines", 20, "ncloc", 30, "coverage", 80).setLanguages(Arrays.asList("java", "python", "kotlin")));
+      newDoc("lines", 10, "ncloc", 20, "coverage", 80)
+        .setLanguages(Arrays.asList("java", "cs", "js"))
+        .setNclocLanguageDistributionFromMap(ImmutableMap.of("java", 200, "cs", 250, "js", 50)),
+      newDoc("lines", 20, "ncloc", 30, "coverage", 80)
+        .setLanguages(Arrays.asList("java", "python", "kotlin"))
+        .setNclocLanguageDistributionFromMap(ImmutableMap.of("java", 300, "python", 100, "kotlin", 404)));
 
     ProjectMeasuresStatistics result = underTest.searchTelemetryStatistics();
 
     assertThat(result.getProjectCount()).isEqualTo(2);
     assertThat(result.getLines()).isEqualTo(30);
     assertThat(result.getNcloc()).isEqualTo(50);
-    assertThat(result.getProjectLanguageDistribution()).containsOnly(
+    assertThat(result.getProjectCountByLanguage()).containsOnly(
       entry("java", 2L), entry("cs", 1L), entry("js", 1L), entry("python", 1L), entry("kotlin", 1L));
+    assertThat(result.getNclocByLanguage()).containsOnly(
+      entry("java", 500L), entry("cs", 250L), entry("js", 50L), entry("python", 100L), entry("kotlin", 404L));
   }
 
   @Test