SONAR-19483 Collect daily counts of analysis and green Quality Gate in Telemetry for branches

Co-authored-by: Zipeng WU <zipeng.wu@sonarsource.com>
Co-authored-by: Nolwenn Cadic <nolwenn.cadic@sonarsource.com>

server/sonar-db-dao/src/it/java/org/sonar/db/component/BranchDaoIT.java

@@ -51,6 +51,7 @@ import static org.apache.commons.lang.StringUtils.repeat;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.entry;
 import static org.assertj.core.api.Assertions.tuple;
+import static org.sonar.api.measures.CoreMetrics.ALERT_STATUS_KEY;
 import static org.sonar.db.component.BranchType.BRANCH;
 import static org.sonar.db.component.BranchType.PULL_REQUEST;
 
@@ -120,6 +121,30 @@ public class BranchDaoIT {
     assertThat(loaded.isMain()).isTrue();
   }
 
+  @Test
+  public void selectBranchMeasuresForTelemetry() {
+    BranchDto dto = new BranchDto();
+    dto.setProjectUuid("U1");
+    dto.setUuid("U1");
+    dto.setBranchType(BranchType.BRANCH);
+    dto.setKey("feature");
+    dto.setIsMain(true);
+    dto.setExcludeFromPurge(false);
+    underTest.insert(dbSession, dto);
+
+    MetricDto qg = db.measures().insertMetric(m -> m.setKey(ALERT_STATUS_KEY));
+    SnapshotDto analysis = db.components().insertSnapshot(dto);
+    db.measures().insertMeasure(dto, analysis, qg, pm -> pm.setData("OK"));
+
+    var branchMeasures = underTest.selectBranchMeasuresWithCaycMetric(dbSession);
+
+    assertThat(branchMeasures)
+      .hasSize(1)
+      .extracting(BranchMeasuresDto::getBranchUuid, BranchMeasuresDto::getBranchKey, BranchMeasuresDto::getProjectUuid,
+        BranchMeasuresDto::getAnalysisCount, BranchMeasuresDto::getGreenQualityGateCount, BranchMeasuresDto::getExcludeFromPurge)
+      .containsExactly(tuple("U1", "feature", "U1", 1, 1, false));
+  }
+
   @Test
   public void updateExcludeFromPurge() {
     BranchDto dto = new BranchDto();
@@ -139,7 +164,7 @@ public class BranchDaoIT {
 
   @DataProvider
   public static Object[][] nullOrEmpty() {
-    return new Object[][]{
+    return new Object[][] {
       {null},
       {""}
     };
@@ -149,7 +174,7 @@ public class BranchDaoIT {
   public static Object[][] oldAndNewValuesCombinations() {
     String value1 = randomAlphabetic(10);
     String value2 = randomAlphabetic(20);
-    return new Object[][]{
+    return new Object[][] {
       {null, value1},
       {"", value1},
       {value1, null},
@@ -454,7 +479,8 @@ public class BranchDaoIT {
 
     assertThat(branches).extracting(BranchDto::getUuid, BranchDto::getKey, BranchDto::isMain, BranchDto::getProjectUuid, BranchDto::getBranchType, BranchDto::getMergeBranchUuid)
       .containsOnly(tuple(mainBranch.getUuid(), mainBranch.getKey(), mainBranch.isMain(), mainBranch.getProjectUuid(), mainBranch.getBranchType(), mainBranch.getMergeBranchUuid()),
-        tuple(featureBranch.getUuid(), featureBranch.getKey(), featureBranch.isMain(), featureBranch.getProjectUuid(), featureBranch.getBranchType(), featureBranch.getMergeBranchUuid()));
+        tuple(featureBranch.getUuid(), featureBranch.getKey(), featureBranch.isMain(), featureBranch.getProjectUuid(), featureBranch.getBranchType(),
+          featureBranch.getMergeBranchUuid()));
   }
 
   @Test
@@ -597,7 +623,8 @@ public class BranchDaoIT {
     db.measures().insertLiveMeasure(project3, unanalyzedC);
 
     assertThat(underTest.countPrBranchAnalyzedLanguageByProjectUuid(db.getSession()))
-      .extracting(PrBranchAnalyzedLanguageCountByProjectDto::getProjectUuid, PrBranchAnalyzedLanguageCountByProjectDto::getBranch, PrBranchAnalyzedLanguageCountByProjectDto::getPullRequest)
+      .extracting(PrBranchAnalyzedLanguageCountByProjectDto::getProjectUuid, PrBranchAnalyzedLanguageCountByProjectDto::getBranch,
+        PrBranchAnalyzedLanguageCountByProjectDto::getPullRequest)
       .containsExactlyInAnyOrder(
         tuple(project1.uuid(), 3L, 3L),
         tuple(project2.uuid(), 1L, 1L),
@@ -805,7 +832,7 @@ public class BranchDaoIT {
 
   @DataProvider
   public static Object[][] booleanValues() {
-    return new Object[][]{
+    return new Object[][] {
       {true},
       {false}
     };

server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java

@@ -19,6 +19,8 @@
  */
 package org.sonar.db.component;
 
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
@@ -199,7 +201,8 @@ public class BranchDao implements Dao {
       .orElse(false);
   }
 
-  public List<BranchDto> selectAllBranches(DbSession dbSession) {
-    return mapper(dbSession).selectAllBranches();
+  public List<BranchMeasuresDto> selectBranchMeasuresWithCaycMetric(DbSession dbSession) {
+    long yesterday = ZonedDateTime.now(ZoneId.systemDefault()).minusDays(1).toInstant().toEpochMilli();
+    return mapper(dbSession).selectBranchMeasuresWithCaycMetric(yesterday);
   }
 }

server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java

@@ -23,7 +23,6 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import org.apache.ibatis.annotations.Param;
 
 public interface BranchMapper {
@@ -77,5 +76,5 @@ public interface BranchMapper {
 
   List<BranchDto> selectMainBranchesByProjectUuids(@Param("projectUuids") Collection<String> projectUuids);
 
-  List<BranchDto> selectAllBranches();
+  List<BranchMeasuresDto> selectBranchMeasuresWithCaycMetric(long yesterday);
 }

server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMeasuresDto.java

@@ -0,0 +1,63 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.component;
+
+public class BranchMeasuresDto {
+  private String branchUuid;
+  private String projectUuid;
+  private String branchKey;
+  private boolean excludeFromPurge;
+  private int greenQualityGateCount;
+  private int analysisCount;
+
+  public BranchMeasuresDto(String branchUuid, String projectUuid, String branchKey, boolean excludeFromPurge, int greenQualityGateCount, int analysisCount) {
+    this.branchUuid = branchUuid;
+    this.projectUuid = projectUuid;
+    this.branchKey = branchKey;
+    this.excludeFromPurge = excludeFromPurge;
+    this.greenQualityGateCount = greenQualityGateCount;
+    this.analysisCount = analysisCount;
+  }
+
+  public String getBranchUuid() {
+    return branchUuid;
+  }
+
+  public String getProjectUuid() {
+    return projectUuid;
+  }
+
+  public boolean getExcludeFromPurge() {
+    return excludeFromPurge;
+  }
+
+  public int getGreenQualityGateCount() {
+    return greenQualityGateCount;
+  }
+
+  public int getAnalysisCount() {
+    return analysisCount;
+  }
+
+  public String getBranchKey() {
+    return branchKey;
+  }
+
+}

server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml

@@ -14,6 +14,15 @@
     pb.is_main as isMain
   </sql>
 
+  <sql id="telemetryColumns">
+    pb.uuid as branchUuid,
+    pb.project_uuid as projectUuid,
+    pb.kee as branchKey,
+    pb.exclude_from_purge as excludeFromPurge,
+    coalesce(ag.greenQualityGateCount, 0) as greenQualityGateCount,
+    coalesce(ag.analysisCount, 0) as analysisCount
+  </sql>
+
   <insert id="insert" parameterType="map" useGeneratedKeys="false">
     insert into project_branches (
     uuid,
@@ -100,11 +109,20 @@
     </foreach>
   </select>
 
-  <select id="selectAllBranches" resultType="org.sonar.db.component.BranchDto">
+  <select id="selectBranchMeasuresWithCaycMetric" resultType="org.sonar.db.component.BranchMeasuresDto">
     select
-    <include refid="columns"/>
+    <include refid="telemetryColumns"/>
     from project_branches pb
-    where branch_type='BRANCH'
+    left join (
+      select pm.component_uuid as branchUuid, count(case when pm.text_value ='OK' then 1 end) as greenQualityGateCount, count(1) as analysisCount
+      from project_measures pm
+      inner join metrics m on m.uuid = pm.metric_uuid
+      inner join snapshots s on s.uuid = pm.analysis_uuid
+      where m.name = 'alert_status' and s.created_at >= #{yesterday, jdbcType=BIGINT}
+      group by pm.component_uuid
+    ) ag
+    on ag.branchUuid = pb.uuid
+    where pb.branch_type='BRANCH'
   </select>
 
   <select id="selectByProjectUuid" parameterType="string" resultType="org.sonar.db.component.BranchDto">

server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java

@@ -328,7 +328,7 @@ public class TelemetryData {
     }
   }
 
-  record Branch(String projectUuid, String branchUuid, int ncdId) {
+  record Branch(String projectUuid, String branchUuid, int ncdId, int greenQualityGateCount, int analysisCount, boolean excludeFromPurge) {
   }
 
   record Project(String projectUuid, Long lastAnalysis, String language, Long loc) {

server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java

@@ -157,6 +157,9 @@ public class TelemetryDataJsonWriter {
         json.prop(PROJECT_ID, branch.projectUuid());
         json.prop("branchUuid", branch.branchUuid());
         json.prop(NCD_ID, branch.ncdId());
+        json.prop("greenQualityGateCount", branch.greenQualityGateCount());
+        json.prop("analysisCount", branch.analysisCount());
+        json.prop("excludeFromPurge", branch.excludeFromPurge());
         json.endObject();
       });
       json.endArray();

server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java

@@ -520,19 +520,24 @@ public class TelemetryDataJsonWriterTest {
       {
         "branches": [
           {
-            "projectUuid": "%s",
-            "branchUuid": "%s",
-            "ncdId": %s
+            "projectUuid": "projectUuid1",
+            "branchUuid": "branchUuid1",
+            "ncdId": 12345,
+            "greenQualityGateCount": 1,
+            "analysisCount": 2,
+            "excludeFromPurge": true
           },
           {
-            "projectUuid": "%s",
-            "branchUuid": "%s",
-            "ncdId": %s
-          },
+            "projectUuid": "projectUuid2",
+            "branchUuid": "branchUuid2",
+            "ncdId": 12345,
+            "greenQualityGateCount": 0,
+            "analysisCount": 2,
+            "excludeFromPurge": true
+          }
         ]
-
       }
-      """.formatted("projectUuid1", "branchUuid1", NCD_ID, "projectUuid2", "branchUuid2", NCD_ID));
+      """);
   }
 
   @Test
@@ -632,9 +637,10 @@ public class TelemetryDataJsonWriterTest {
   }
 
   private List<TelemetryData.Branch> attachBranches() {
-    return List.of(new TelemetryData.Branch("projectUuid1", "branchUuid1", NCD_ID),
-      new TelemetryData.Branch("projectUuid2", "branchUuid2", NCD_ID));
+    return List.of(new TelemetryData.Branch("projectUuid1", "branchUuid1", NCD_ID, 1, 2, true),
+      new TelemetryData.Branch("projectUuid2", "branchUuid2", NCD_ID, 0, 2, true));
   }
+
   private List<TelemetryData.NewCodeDefinition> attachNewCodeDefinitions() {
     return List.of(NCD_INSTANCE, NCD_PROJECT);
   }

server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java

@@ -48,7 +48,7 @@ import org.sonar.db.DbSession;
 import org.sonar.db.alm.setting.ALM;
 import org.sonar.db.alm.setting.ProjectAlmKeyAndProject;
 import org.sonar.db.component.AnalysisPropertyValuePerProject;
-import org.sonar.db.component.BranchDto;
+import org.sonar.db.component.BranchMeasuresDto;
 import org.sonar.db.component.PrBranchAnalyzedLanguageCountByProjectDto;
 import org.sonar.db.component.SnapshotDto;
 import org.sonar.db.measure.LiveMeasureDto;
@@ -153,8 +153,8 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
       getVersion));
     data.setPlugins(plugins);
     try (DbSession dbSession = dbClient.openSession(false)) {
-      var branchDtos = dbClient.branchDao().selectAllBranches(dbSession);
-      loadNewCodeDefinitions(dbSession, branchDtos);
+      var branchMeasuresDtos = dbClient.branchDao().selectBranchMeasuresWithCaycMetric(dbSession);
+      loadNewCodeDefinitions(dbSession, branchMeasuresDtos);
 
       data.setDatabase(loadDatabaseMetadata(dbSession));
       data.setNcdId(instanceNcd.hashCode());
@@ -166,7 +166,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
       resolveUnanalyzedLanguageCode(data, dbSession);
       resolveProjectStatistics(data, dbSession, defaultQualityGateUuid);
       resolveProjects(data, dbSession);
-      resolveBranches(data, branchDtos);
+      resolveBranches(data, branchMeasuresDtos);
       resolveQualityGates(data, dbSession);
       resolveUsers(data, dbSession);
     }
@@ -184,12 +184,14 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
       .build();
   }
 
-  private void resolveBranches(TelemetryData.Builder data, List<BranchDto> branchDtos) {
-    var branches = branchDtos.stream()
+  private void resolveBranches(TelemetryData.Builder data, List<BranchMeasuresDto> branchMeasuresDtos) {
+    var branches = branchMeasuresDtos.stream()
       .map(dto -> {
         var projectNcd = ncdByProject.getOrDefault(dto.getProjectUuid(), instanceNcd);
-        var ncdId = ncdByBranch.getOrDefault(dto.getUuid(), projectNcd).hashCode();
-        return new TelemetryData.Branch(dto.getProjectUuid(), dto.getUuid(), ncdId);
+        var ncdId = ncdByBranch.getOrDefault(dto.getBranchUuid(), projectNcd).hashCode();
+        return new TelemetryData.Branch(
+          dto.getProjectUuid(), dto.getBranchUuid(), ncdId,
+          dto.getGreenQualityGateCount(), dto.getAnalysisCount(), dto.getExcludeFromPurge());
       })
       .toList();
     data.setBranches(branches);
@@ -203,8 +205,9 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
     this.instanceNcd = NewCodeDefinition.getInstanceDefault();
   }
 
-  private void loadNewCodeDefinitions(DbSession dbSession, List<BranchDto> branchDtos) {
-    var branchUuidByKey = branchDtos.stream().collect(Collectors.toMap(dto -> createBranchUniqueKey(dto.getProjectUuid(), dto.getBranchKey()), BranchDto::getUuid));
+  private void loadNewCodeDefinitions(DbSession dbSession, List<BranchMeasuresDto> branchMeasuresDtos) {
+    var branchUuidByKey = branchMeasuresDtos.stream()
+      .collect(Collectors.toMap(dto -> createBranchUniqueKey(dto.getProjectUuid(), dto.getBranchKey()), BranchMeasuresDto::getBranchUuid));
     List<NewCodePeriodDto> newCodePeriodDtos = dbClient.newCodePeriodDao().selectAll(dbSession);
     NewCodeDefinition ncd;
     boolean hasInstance = false;

server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java

@@ -24,6 +24,9 @@ import com.tngtech.java.junit.dataprovider.DataProviderRunner;
 import com.tngtech.java.junit.dataprovider.UseDataProvider;
 import java.sql.DatabaseMetaData;
 import java.sql.SQLException;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
@@ -59,6 +62,7 @@ import org.sonar.server.property.InternalProperties;
 import org.sonar.server.property.MapInternalProperties;
 import org.sonar.server.qualitygate.QualityGateCaycChecker;
 import org.sonar.server.qualitygate.QualityGateFinder;
+import org.sonar.server.telemetry.TelemetryData.Branch;
 import org.sonar.server.telemetry.TelemetryData.NewCodeDefinition;
 import org.sonar.server.telemetry.TelemetryData.ProjectStatistics;
 import org.sonar.updatecenter.common.Version;
@@ -72,6 +76,7 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
+import static org.sonar.api.measures.CoreMetrics.ALERT_STATUS_KEY;
 import static org.sonar.api.measures.CoreMetrics.BUGS_KEY;
 import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY;
 import static org.sonar.api.measures.CoreMetrics.DEVELOPMENT_COST_KEY;
@@ -251,7 +256,7 @@ public class TelemetryDataLoaderImplTest {
           Optional.empty(), instanceNcdId));
 
     assertThat(data.getBranches())
-      .extracting(TelemetryData.Branch::branchUuid, TelemetryData.Branch::ncdId)
+      .extracting(Branch::branchUuid, Branch::ncdId)
       .containsExactlyInAnyOrder(
         tuple(branch1.uuid(), projectNcdId),
         tuple(branch2.uuid(), branchNcdId),
@@ -274,6 +279,45 @@ public class TelemetryDataLoaderImplTest {
       );
   }
 
+  @Test
+  public void send_branch_measures_data() {
+    Long analysisDate = ZonedDateTime.now(ZoneId.systemDefault()).toInstant().toEpochMilli();
+
+    MetricDto qg = db.measures().insertMetric(m -> m.setKey(ALERT_STATUS_KEY));
+
+    ComponentDto project1 = db.components().insertPrivateProject().getMainBranchComponent();
+
+    ComponentDto project2 = db.components().insertPrivateProject().getMainBranchComponent();
+
+    SnapshotDto project1Analysis1 = db.components().insertSnapshot(project1, t -> t.setLast(true).setBuildDate(analysisDate));
+    SnapshotDto project1Analysis2 = db.components().insertSnapshot(project1, t -> t.setLast(true).setBuildDate(analysisDate));
+    SnapshotDto project2Analysis = db.components().insertSnapshot(project2, t -> t.setLast(true).setBuildDate(analysisDate));
+    db.measures().insertMeasure(project1, project1Analysis1, qg, pm -> pm.setData("OK"));
+    db.measures().insertMeasure(project1, project1Analysis2, qg, pm -> pm.setData("ERROR"));
+    db.measures().insertMeasure(project2, project2Analysis, qg, pm -> pm.setData("ERROR"));
+
+    var branch1 = db.components().insertProjectBranch(project1, branchDto -> branchDto.setKey("reference"));
+    var branch2 = db.components().insertProjectBranch(project1, branchDto -> branchDto.setKey("custom"));
+
+    db.newCodePeriods().insert(project1.uuid(), NewCodePeriodType.NUMBER_OF_DAYS, "30");
+    db.newCodePeriods().insert(project1.uuid(), branch2.branchUuid(), NewCodePeriodType.REFERENCE_BRANCH, "reference");
+
+    var instanceNcdId = NewCodeDefinition.getInstanceDefault().hashCode();
+    var projectNcdId = new NewCodeDefinition(NewCodePeriodType.NUMBER_OF_DAYS.name(), "30", "project").hashCode();
+    var branchNcdId = new NewCodeDefinition(NewCodePeriodType.REFERENCE_BRANCH.name(), branch1.uuid(), "branch").hashCode();
+
+    TelemetryData data = communityUnderTest.load();
+
+    assertThat(data.getBranches())
+      .extracting(Branch::branchUuid, Branch::ncdId, Branch::greenQualityGateCount, Branch::analysisCount)
+      .containsExactlyInAnyOrder(
+        tuple(branch1.uuid(), projectNcdId, 0, 0),
+        tuple(branch2.uuid(), branchNcdId, 0, 0),
+        tuple(project1.uuid(), projectNcdId, 1, 2),
+        tuple(project2.uuid(), instanceNcdId, 0, 1));
+
+  }
+
   private List<UserDto> composeActiveUsers(int count) {
     UserDbTester userDbTester = db.users();
     Function<Integer, Consumer<UserDto>> userConfigurator = index -> user -> user.setExternalIdentityProvider("provider" + index).setLastSonarlintConnectionDate(index * 2L);
@@ -355,7 +399,7 @@ public class TelemetryDataLoaderImplTest {
       .containsExactlyInAnyOrder(tuple(2L, projectNcdId));
 
     assertThat(data.getBranches())
-      .extracting(TelemetryData.Branch::branchUuid, TelemetryData.Branch::ncdId)
+      .extracting(Branch::branchUuid, Branch::ncdId)
       .contains(tuple(branch.uuid(), projectNcdId));
   }