SONAR-18472 Resolve telemetry performance issue

Co-authored-by: Jacek Poreda <jacek.poreda@sonarsource.com>
(cherry picked from commit a20f2bce3c)

server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java

@@ -81,7 +81,7 @@ import org.sonar.db.measure.LargestBranchNclocDto;
 import org.sonar.db.measure.LiveMeasureMapper;
 import org.sonar.db.measure.MeasureDto;
 import org.sonar.db.measure.MeasureMapper;
-import org.sonar.db.measure.ProjectMeasureDto;
+import org.sonar.db.measure.ProjectLocDistributionDto;
 import org.sonar.db.metric.MetricMapper;
 import org.sonar.db.newcodeperiod.NewCodePeriodMapper;
 import org.sonar.db.notification.NotificationQueueDto;
@@ -225,7 +225,7 @@ public class MyBatis {
     confBuilder.loadAlias("ProjectAlmKeyAndProject", ProjectAlmKeyAndProject.class);
     confBuilder.loadAlias("PrAndBranchCountByProjectDto", PrBranchAnalyzedLanguageCountByProjectDto.class);
     confBuilder.loadAlias("ProjectMapping", ProjectMappingDto.class);
-    confBuilder.loadAlias("ProjectMeasure", ProjectMeasureDto.class);
+    confBuilder.loadAlias("ProjectLocDistribution", ProjectLocDistributionDto.class);
     confBuilder.loadAlias("PurgeableAnalysis", PurgeableAnalysisDto.class);
     confBuilder.loadAlias("PushEvent", PushEventDto.class);
     confBuilder.loadAlias("QualityGateCondition", QualityGateConditionDto.class);

server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java

@@ -109,6 +109,10 @@ public class LiveMeasureDao implements Dao {
     return mapper(dbSession).getLargestBranchNclocPerProject();
   }
 
+  public List<ProjectLocDistributionDto> selectLargestBranchesLocDistribution(DbSession session, String nclocUuid, String nclocDistributionUuid) {
+    return mapper(session).selectLargestBranchesLocDistribution(nclocUuid, nclocDistributionUuid);
+  }
+
   public long countProjectsHavingMeasure(DbSession dbSession, String metric) {
     return mapper(dbSession).countProjectsHavingMeasure(metric);
   }

server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java

@@ -64,6 +64,8 @@ public interface LiveMeasureMapper {
 
   List<LargestBranchNclocDto> getLargestBranchNclocPerProject();
 
+  List<ProjectLocDistributionDto> selectLargestBranchesLocDistribution(@Param("nclocUuid") String nclocUuid, @Param("nclocDistributionUuid") String nclocDistributionUuid);
+
   Long countProjectsHavingMeasure(
     @Param("metric") String metric);
 

server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java

@@ -76,8 +76,4 @@ public class MeasureDao implements Dao {
     return session.getMapper(MeasureMapper.class);
   }
 
-  public List<ProjectMeasureDto> selectLastMeasureForAllProjects(DbSession session, String metricKey) {
-    return mapper(session).selectLastMeasureForAllProjects(metricKey);
-
-  }
 }

server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java

@@ -42,5 +42,4 @@ public interface MeasureMapper {
 
   void insert(MeasureDto measureDto);
 
-  List<ProjectMeasureDto> selectLastMeasureForAllProjects(@Param("metricKey") String metricKey);
 }

server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasureDto.java → server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectLocDistributionDto.java

@@ -19,47 +19,9 @@
  */
 package org.sonar.db.measure;
 
-public class ProjectMeasureDto {
-
-  private String projectUuid;
-  private Long lastAnalysis;
-  private long loc;
-  private String textValue;
-
-  public String getProjectUuid() {
-    return projectUuid;
-  }
-
-  public ProjectMeasureDto setProjectUuid(String projectUuid) {
-    this.projectUuid = projectUuid;
-    return this;
-  }
-
-  public String getTextValue() {
-    return textValue;
-  }
-
-  public ProjectMeasureDto setTextValue(String textValue) {
-    this.textValue = textValue;
-    return this;
-  }
-
-  public long getLoc() {
-    return loc;
-  }
-
-  public ProjectMeasureDto setLoc(long loc) {
-    this.loc = loc;
-    return this;
-  }
-
-  public Long getLastAnalysis() {
-    return lastAnalysis;
-  }
-
-  public ProjectMeasureDto setLastAnalysis(Long lastAnalysis) {
-    this.lastAnalysis = lastAnalysis;
-    return this;
-  }
+/**
+ * Loc distribution per language for the largest branch in a project.
+ */
+public record ProjectLocDistributionDto(String projectUuid, String branchUuid, String locDistribution) {
 
 }

server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml

@@ -112,6 +112,24 @@
       order by ncloc desc
   </select>
 
+  <select id="selectLargestBranchesLocDistribution" parameterType="map" resultType="ProjectLocDistribution">
+    select top_branches.project_uuid as projectUuid,
+    top_branches.uuid as branchUuid,
+    lm2.text_value as locDistribution from (
+    SELECT loc_grouped_branches.uuid, loc_grouped_branches.project_uuid
+      FROM (
+       SELECT b.uuid, b.project_uuid, ROW_NUMBER() OVER (PARTITION BY
+       b.project_uuid ORDER BY lm.value desc, b.uuid asc) row_number
+       from live_measures lm
+          inner join project_branches b on b.uuid = lm.component_uuid
+          inner join projects p on p.uuid = b.project_uuid
+          where lm.metric_uuid = #{nclocUuid, jdbcType=VARCHAR}
+          and p.qualifier ='TRK'
+      ) loc_grouped_branches
+    WHERE loc_grouped_branches.row_number = 1) top_branches
+    inner join live_measures lm2 on lm2.component_uuid = top_branches.uuid
+    where lm2.metric_uuid = #{nclocDistributionUuid, jdbcType=VARCHAR}
+  </select>
 
   <select id="countProjectsHavingMeasure" parameterType="map" resultType="long">
     select count(1)

server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml

@@ -26,41 +26,6 @@
     s.islast= ${_true}
   </select>
 
-
-  <select id="selectLastMeasureForAllProjects" parameterType="map" resultType="ProjectMeasure">
-    select tie_breaker.projectUuid as projectUuid,
-    s.build_date as lastAnalysis,
-    ncloc as loc,
-    pm.text_value as textValue
-      from
-        (select counter.projectUuid as projectUuid,
-          counter.maxncloc ncloc,
-          min(br.uuid) as branchUuid,
-          counter.projectName,
-          counter.projectKey
-          from
-            (select b.project_uuid as projectUuid,
-              p.name as projectName,
-              p.kee as projectKey,
-              max(lm.value) as maxncloc
-                from live_measures lm
-                inner join metrics m on m.uuid = lm.metric_uuid
-                inner join project_branches b on b.uuid = lm.component_uuid
-                inner join projects p on p.uuid = b.project_uuid and p.qualifier = 'TRK'
-                where m.name = 'ncloc'
-                group by b.project_uuid, p.name, p.kee) counter
-          inner join live_measures lmo on lmo.value = counter.maxncloc
-          inner join project_branches br on br.project_uuid = counter.projectUuid and br.uuid = lmo.component_uuid
-          inner join components c on c.uuid = br.uuid
-          group by counter.projectUuid, counter.maxncloc, counter.projectName, counter.projectKey) tie_breaker
-      inner join project_branches pb on tie_breaker.branchUuid = pb.uuid
-      inner join project_measures pm on pb.uuid = pm.component_uuid
-      inner join snapshots s on s.component_uuid = pb.uuid and s.islast = ${_true}
-      inner join metrics m on m.name = #{metricKey,jdbcType=VARCHAR} and m.uuid = pm.metric_uuid
-      where pm.analysis_uuid = s.uuid
-      order by ncloc desc
-  </select>
-
   <select id="selectMeasure" parameterType="map" resultType="Measure">
     select <include refid="measureColumns"/>
     from project_measures pm

server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java

@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.stream.IntStream;
 import org.apache.commons.lang.RandomStringUtils;
@@ -43,6 +44,7 @@ import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.groups.Tuple.tuple;
+import static org.sonar.api.measures.Metric.ValueType.DATA;
 import static org.sonar.api.measures.Metric.ValueType.INT;
 import static org.sonar.db.component.BranchDto.DEFAULT_MAIN_BRANCH_NAME;
 import static org.sonar.db.component.ComponentTesting.newFileDto;
@@ -373,7 +375,8 @@ public class LiveMeasureDaoTest {
 
   @Test
   public void get_branch_with_max_ncloc_per_project() {
-    setupProjectsWithLoc();
+    Map<String, MetricDto> metrics = setupMetrics();
+    setupProjectsWithLoc(metrics.get("ncloc"), metrics.get("ncloc_language_distribution"), metrics.get("lines"));
 
     List<LargestBranchNclocDto> results = underTest.getLargestBranchNclocPerProject(db.getSession());
 
@@ -385,6 +388,24 @@ public class LiveMeasureDaoTest {
     assertLocForProject(results.get(4), "projectWithLinesButNoLoc", DEFAULT_MAIN_BRANCH_NAME, 0);
   }
 
+  @Test
+  public void get_loc_language_distribution() {
+    Map<String, MetricDto> metrics = setupMetrics();
+    MetricDto ncloc = metrics.get("ncloc");
+    MetricDto nclocLanguageDistribution = metrics.get("ncloc_language_distribution");
+    Map<String, ComponentDto> components = setupProjectsWithLoc(ncloc, nclocLanguageDistribution, metrics.get("lines"));
+
+    List<ProjectLocDistributionDto> results = underTest.selectLargestBranchesLocDistribution(db.getSession(), ncloc.getUuid(), nclocLanguageDistribution.getUuid());
+
+    assertThat(results)
+      .containsExactlyInAnyOrder(
+        new ProjectLocDistributionDto(components.get("projectWithTieOnBranchSize").uuid(), components.get("projectWithTieOnBranchSize").uuid(), "java=250;js=0"),
+        new ProjectLocDistributionDto(components.get("projectWithTieOnOtherBranches").uuid(), components.get("tieBranch1").uuid(), "java=230;js=0"),
+        new ProjectLocDistributionDto(components.get("projectWithBranchBiggerThanMaster").uuid(), components.get("notMasterBranch").uuid(), "java=100;js=100"),
+        new ProjectLocDistributionDto(components.get("simpleProject").uuid(), components.get("simpleProject").uuid(), "java=10;js=0"),
+        new ProjectLocDistributionDto(components.get("projectWithLinesButNoLoc").uuid(), components.get("projectWithLinesButNoLoc").uuid(), "java=0;js=0"));
+  }
+
   @Test
   public void countNcloc_empty() {
     db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString()));
@@ -696,29 +717,55 @@ public class LiveMeasureDaoTest {
       "componentUuid", "projectUuid", "metricUuid", "value", "textValue", "data");
   }
 
-  private void setupProjectsWithLoc() {
+  private Map<String, MetricDto> setupMetrics() {
     MetricDto ncloc = db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString()));
+    MetricDto nclocDistribution = db.measures().insertMetric(m -> m.setKey("ncloc_language_distribution").setValueType(DATA.toString()));
     MetricDto lines = db.measures().insertMetric(m -> m.setKey("lines").setValueType(INT.toString()));
+    return Map.of("ncloc", ncloc,
+      "ncloc_language_distribution", nclocDistribution,
+      "lines", lines);
+  }
 
-    addProjectWithMeasure("simpleProject", ncloc, 10d);
+  private Map<String, ComponentDto> setupProjectsWithLoc(MetricDto ncloc, MetricDto nclocDistribution, MetricDto lines) {
+    ComponentDto simpleProject = addProjectWithMeasure("simpleProject", ncloc, 10d);
+    addMeasureToComponent(simpleProject, nclocDistribution, "java=10;js=0");
 
     ComponentDto projectWithBranchBiggerThanMaster = addProjectWithMeasure("projectWithBranchBiggerThanMaster", ncloc, 100d);
-    addBranchToProjectWithMeasure(projectWithBranchBiggerThanMaster,"notMasterBranch", ncloc, 200d);
+    addMeasureToComponent(projectWithBranchBiggerThanMaster, nclocDistribution, "java=100;js=0");
+
+    ComponentDto notMasterBranch = addBranchToProjectWithMeasure(projectWithBranchBiggerThanMaster, "notMasterBranch", ncloc, 200d);
+    addMeasureToComponent(notMasterBranch, nclocDistribution, "java=100;js=100");
 
     ComponentDto projectWithLinesButNoLoc = addProjectWithMeasure("projectWithLinesButNoLoc", lines, 365d);
-    addMeasureToComponent(projectWithLinesButNoLoc,ncloc,0d,false);
+    addMeasureToComponent(projectWithLinesButNoLoc, nclocDistribution, "java=0;js=0");
+    addMeasureToComponent(projectWithLinesButNoLoc, ncloc, 0d, false);
 
     ComponentDto projectWithTieOnBranchSize = addProjectWithMeasure("projectWithTieOnBranchSize", ncloc, 250d);
-    addBranchToProjectWithMeasure(projectWithTieOnBranchSize,"tieBranch", ncloc, 250d);
+    addMeasureToComponent(projectWithTieOnBranchSize, nclocDistribution, "java=250;js=0");
+    ComponentDto tieBranch = addBranchToProjectWithMeasure(projectWithTieOnBranchSize, "tieBranch", ncloc, 250d);
+    addMeasureToComponent(tieBranch, nclocDistribution, "java=250;js=0");
 
     ComponentDto projectWithTieOnOtherBranches = addProjectWithMeasure("projectWithTieOnOtherBranches", ncloc, 220d);
-    addBranchToProjectWithMeasure(projectWithTieOnOtherBranches,"tieBranch1", ncloc, 230d);
-    addBranchToProjectWithMeasure(projectWithTieOnOtherBranches,"tieBranch2", ncloc, 230d);
+    addMeasureToComponent(projectWithTieOnOtherBranches, nclocDistribution, "java=220;js=0");
+    ComponentDto tieBranch1 = addBranchToProjectWithMeasure(projectWithTieOnOtherBranches, "tieBranch1", ncloc, 230d);
+    addMeasureToComponent(tieBranch1, nclocDistribution, "java=230;js=0");
+    ComponentDto tieBranch2 = addBranchToProjectWithMeasure(projectWithTieOnOtherBranches, "tieBranch2", ncloc, 230d);
+    addMeasureToComponent(tieBranch2, nclocDistribution, "java=230;js=0");
+
+    return Map.of("simpleProject", simpleProject,
+      "projectWithBranchBiggerThanMaster", projectWithBranchBiggerThanMaster,
+      "notMasterBranch", notMasterBranch,
+      "projectWithLinesButNoLoc", projectWithLinesButNoLoc,
+      "projectWithTieOnBranchSize", projectWithTieOnBranchSize,
+      "tieBranch", tieBranch,
+      "projectWithTieOnOtherBranches", projectWithTieOnOtherBranches,
+      "tieBranch1", tieBranch1,
+      "tieBranch2", tieBranch2);
   }
 
   private ComponentDto addProjectWithMeasure(String projectKey, MetricDto metric, double metricValue) {
     ComponentDto project = db.components().insertPublicProject(p -> p.setKey(projectKey));
-    addMeasureToComponent(project, metric, metricValue,true);
+    addMeasureToComponent(project, metric, metricValue, true);
     return project;
   }
 
@@ -729,9 +776,14 @@ public class LiveMeasureDaoTest {
     }
   }
 
-  private void addBranchToProjectWithMeasure(ComponentDto project, String branchKey, MetricDto metric, double metricValue) {
+  private void addMeasureToComponent(ComponentDto component, MetricDto metric, String metricValue) {
+    db.measures().insertLiveMeasure(component, metric, m -> m.setData(metricValue));
+  }
+
+  private ComponentDto addBranchToProjectWithMeasure(ComponentDto project, String branchKey, MetricDto metric, double metricValue) {
     ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH).setKey(branchKey));
-    addMeasureToComponent(branch, metric, metricValue,true);
+    addMeasureToComponent(branch, metric, metricValue, true);
+    return branch;
   }
 
   private void assertLocForProject(LargestBranchNclocDto result, String projectKey, String branchKey, long linesOfCode) {

server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureDaoTest.java

@@ -46,6 +46,7 @@ public class MeasureDaoTest {
   private MetricDto coverage;
   private MetricDto complexity;
   private MetricDto ncloc;
+  private MetricDto nclocDistribution;
 
 
   @Rule
@@ -60,6 +61,7 @@ public class MeasureDaoTest {
     coverage =db.measures().insertMetric(m -> m.setKey("coverage"));
     complexity = db.measures().insertMetric(m -> m.setKey( "complexity"));
     ncloc = db.measures().insertMetric(m -> m.setKey("ncloc"));
+    nclocDistribution = db.measures().insertMetric(m -> m.setKey("ncloc_language_distribution"));
   }
 
   @Test

server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java

@@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting;
 import java.sql.DatabaseMetaData;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -46,8 +47,9 @@ import org.sonar.db.alm.setting.ALM;
 import org.sonar.db.alm.setting.ProjectAlmKeyAndProject;
 import org.sonar.db.component.AnalysisPropertyValuePerProject;
 import org.sonar.db.component.PrBranchAnalyzedLanguageCountByProjectDto;
+import org.sonar.db.component.SnapshotDto;
 import org.sonar.db.measure.LiveMeasureDto;
-import org.sonar.db.measure.ProjectMeasureDto;
+import org.sonar.db.measure.ProjectLocDistributionDto;
 import org.sonar.db.metric.MetricDto;
 import org.sonar.db.qualitygate.ProjectQgateAssociationDto;
 import org.sonar.db.qualitygate.QualityGateDto;
@@ -64,6 +66,7 @@ import static java.util.stream.Collectors.toMap;
 import static org.sonar.api.internal.apachecommons.lang.StringUtils.startsWithIgnoreCase;
 import static org.sonar.api.measures.CoreMetrics.BUGS_KEY;
 import static org.sonar.api.measures.CoreMetrics.DEVELOPMENT_COST_KEY;
+import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
 import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY;
 import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_KEY;
 import static org.sonar.api.measures.CoreMetrics.TECHNICAL_DEBT_KEY;
@@ -222,17 +225,35 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
   }
 
   private void resolveProjects(TelemetryData.Builder data, DbSession dbSession) {
-    List<ProjectMeasureDto> measures = dbClient.measureDao().selectLastMeasureForAllProjects(dbSession, NCLOC_LANGUAGE_DISTRIBUTION_KEY);
-    List<TelemetryData.Project> projects = new ArrayList<>();
-    for (ProjectMeasureDto measure : measures) {
-      for (String measureTextValue : measure.getTextValue().split(";")) {
-        String[] languageAndLoc = measureTextValue.split("=");
-        String language = languageAndLoc[0];
-        Long loc = Long.parseLong(languageAndLoc[1]);
-        projects.add(new TelemetryData.Project(measure.getProjectUuid(), measure.getLastAnalysis(), language, loc));
-      }
-    }
-    data.setProjects(projects);
+    Map<String, String> metricUuidMap = getNclocMetricUuidMap(dbSession);
+    String nclocUuid = metricUuidMap.get(NCLOC_KEY);
+    String nclocDistributionUuid = metricUuidMap.get(NCLOC_LANGUAGE_DISTRIBUTION_KEY);
+    List<ProjectLocDistributionDto> branchesWithLargestNcloc = dbClient.liveMeasureDao().selectLargestBranchesLocDistribution(dbSession, nclocUuid, nclocDistributionUuid);
+    List<String> branchUuids = branchesWithLargestNcloc.stream().map(ProjectLocDistributionDto::branchUuid).toList();
+    Map<String, Long> latestSnapshotMap = dbClient.snapshotDao().selectLastAnalysesByRootComponentUuids(dbSession, branchUuids)
+      .stream()
+      .collect(toMap(SnapshotDto::getComponentUuid, SnapshotDto::getBuildDate));
+    data.setProjects(buildProjectsList(branchesWithLargestNcloc, latestSnapshotMap));
+  }
+
+  private static List<TelemetryData.Project> buildProjectsList(List<ProjectLocDistributionDto> branchesWithLargestNcloc,
+    Map<String, Long> latestSnapshotMap) {
+    return branchesWithLargestNcloc.stream()
+      .flatMap(measure -> Arrays.stream(measure.locDistribution().split(";"))
+        .map(languageAndLoc -> languageAndLoc.split("="))
+        .map(languageAndLoc -> new TelemetryData.Project(
+          measure.projectUuid(),
+          latestSnapshotMap.get(measure.branchUuid()),
+          languageAndLoc[0],
+          Long.parseLong(languageAndLoc[1])
+        ))
+      ).toList();
+  }
+
+  private Map<String, String> getNclocMetricUuidMap(DbSession dbSession) {
+    return dbClient.metricDao().selectByKeys(dbSession, asList(NCLOC_KEY, NCLOC_LANGUAGE_DISTRIBUTION_KEY))
+      .stream()
+      .collect(toMap(MetricDto::getKey, MetricDto::getUuid));
   }
 
   private void resolveQualityGates(TelemetryData.Builder data, DbSession dbSession) {