SONAR-19297 Track new code definition in Telemetry

Co-authored-by: Zipeng WU <zipeng.wu@sonarsource.com>
Co-authored-by: Nolwenn Cadic <nolwenn.cadic@sonarsource.com>

3 files changed, 117 insertions, 17 deletions

diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java
index 974219952ba..fe9ad60ea11 100644
--- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java
+++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java
@@ -150,6 +150,7 @@ public class TelemetryDaemon extends AbstractStoppableScheduledExecutorServiceIm
       dataJsonWriter.writeTelemetryData(json, statistics);
     }
     telemetryClient.upload(jsonString.toString());
+    dataLoader.reset();
   }
 
   private boolean shouldUploadStatistics(long now) {
diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java
index 0d87f2b9e0b..03c441fe5b7 100644
--- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java
+++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java
@@ -27,12 +27,14 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 import javax.inject.Inject;
 import org.sonar.api.config.Configuration;
 import org.sonar.api.platform.Server;
@@ -46,11 +48,13 @@ import org.sonar.db.DbSession;
 import org.sonar.db.alm.setting.ALM;
 import org.sonar.db.alm.setting.ProjectAlmKeyAndProject;
 import org.sonar.db.component.AnalysisPropertyValuePerProject;
+import org.sonar.db.component.BranchDto;
 import org.sonar.db.component.PrBranchAnalyzedLanguageCountByProjectDto;
 import org.sonar.db.component.SnapshotDto;
 import org.sonar.db.measure.LiveMeasureDto;
 import org.sonar.db.measure.ProjectLocDistributionDto;
 import org.sonar.db.metric.MetricDto;
+import org.sonar.db.newcodeperiod.NewCodePeriodDto;
 import org.sonar.db.qualitygate.ProjectQgateAssociationDto;
 import org.sonar.db.qualitygate.QualityGateDto;
 import org.sonar.server.management.ManagedInstanceService;
@@ -59,6 +63,7 @@ import org.sonar.server.property.InternalProperties;
 import org.sonar.server.qualitygate.QualityGateCaycChecker;
 import org.sonar.server.qualitygate.QualityGateFinder;
 import org.sonar.server.telemetry.TelemetryData.Database;
+import org.sonar.server.telemetry.TelemetryData.NewCodeDefinition;
 
 import static java.util.Arrays.asList;
 import static java.util.Optional.ofNullable;
@@ -77,6 +82,7 @@ import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETEC
 import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY;
 import static org.sonar.core.platform.EditionProvider.Edition.DATACENTER;
 import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE;
+import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH;
 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY;
 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY;
 import static org.sonar.server.telemetry.TelemetryDaemon.I_PROP_MESSAGE_SEQUENCE;
@@ -103,6 +109,10 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
   private final QualityGateCaycChecker qualityGateCaycChecker;
   private final QualityGateFinder qualityGateFinder;
   private final ManagedInstanceService managedInstanceService;
+  private final Set<NewCodeDefinition> newCodeDefinitions = new HashSet<>();
+  private final Map<String, NewCodeDefinition> ncdByProject = new HashMap<>();
+  private final Map<String, NewCodeDefinition> ncdByBranch = new HashMap<>();
+  private NewCodeDefinition instanceNcd = NewCodeDefinition.getInstanceDefault();
 
   @Inject
   public TelemetryDataLoaderImpl(Server server, DbClient dbClient, PluginRepository pluginRepository,
@@ -143,7 +153,12 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
       getVersion));
     data.setPlugins(plugins);
     try (DbSession dbSession = dbClient.openSession(false)) {
+      var branchDtos = dbClient.branchDao().selectAllBranches(dbSession);
+      loadNewCodeDefinitions(dbSession, branchDtos);
+
       data.setDatabase(loadDatabaseMetadata(dbSession));
+      data.setNcdId(instanceNcd.hashCode());
+      data.setNewCodeDefinitions(newCodeDefinitions);
 
       String defaultQualityGateUuid = qualityGateFinder.getDefault(dbSession).getUuid();
 
@@ -151,6 +166,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
       resolveUnanalyzedLanguageCode(data, dbSession);
       resolveProjectStatistics(data, dbSession, defaultQualityGateUuid);
       resolveProjects(data, dbSession);
+      resolveBranches(data, branchDtos);
       resolveQualityGates(data, dbSession);
       resolveUsers(data, dbSession);
     }
@@ -161,7 +177,6 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
     installationDateProperty.ifPresent(s -> data.setInstallationDate(Long.valueOf(s)));
     Optional<String> installationVersionProperty = internalProperties.read(InternalProperties.INSTALLATION_VERSION);
 
-
     return data
       .setInstallationVersion(installationVersionProperty.orElse(null))
       .setInDocker(dockerSupport.isRunningInDocker())
@@ -169,6 +184,59 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
       .build();
   }
 
+  private void resolveBranches(TelemetryData.Builder data, List<BranchDto> branchDtos) {
+    var branches = branchDtos.stream()
+      .map(dto -> {
+        var projectNcd = ncdByProject.getOrDefault(dto.getProjectUuid(), instanceNcd);
+        var ncdId = ncdByBranch.getOrDefault(dto.getUuid(), projectNcd).hashCode();
+        return new TelemetryData.Branch(dto.getProjectUuid(), dto.getUuid(), ncdId);
+      })
+      .toList();
+    data.setBranches(branches);
+  }
+
+  @Override
+  public void reset() {
+    this.newCodeDefinitions.clear();
+    this.ncdByBranch.clear();
+    this.ncdByProject.clear();
+    this.instanceNcd = NewCodeDefinition.getInstanceDefault();
+  }
+
+  private void loadNewCodeDefinitions(DbSession dbSession, List<BranchDto> branchDtos) {
+    var branchUuidByKey = branchDtos.stream().collect(Collectors.toMap(dto -> createBranchUniqueKey(dto.getProjectUuid(), dto.getBranchKey()), BranchDto::getUuid));
+    List<NewCodePeriodDto> newCodePeriodDtos = dbClient.newCodePeriodDao().selectAll(dbSession);
+    NewCodeDefinition ncd;
+    boolean hasInstance = false;
+    for (var dto : newCodePeriodDtos) {
+      String projectUuid = dto.getProjectUuid();
+      String branchUuid = dto.getBranchUuid();
+      if (branchUuid == null && projectUuid == null) {
+        ncd = new NewCodeDefinition(dto.getType().name(), dto.getValue(), "instance");
+        this.instanceNcd = ncd;
+        hasInstance = true;
+      } else if (projectUuid != null) {
+        var value = dto.getType() == REFERENCE_BRANCH ? branchUuidByKey.get(createBranchUniqueKey(projectUuid, dto.getValue())) : dto.getValue();
+        if (branchUuid == null) {
+          ncd = new NewCodeDefinition(dto.getType().name(), value, "project");
+          this.ncdByProject.put(projectUuid, ncd);
+        } else {
+          ncd = new NewCodeDefinition(dto.getType().name(), value, "branch");
+          this.ncdByBranch.put(branchUuid, ncd);
+        }
+      } else {
+        throw new IllegalStateException(String.format("Error in loading telemetry data. New code definition for branch %s doesn't have a projectUuid", branchUuid));
+      }
+      this.newCodeDefinitions.add(ncd);
+    }
+    if (!hasInstance) {
+      this.newCodeDefinitions.add(NewCodeDefinition.getInstanceDefault());
+    }
+  }
+
+  private static String createBranchUniqueKey(String projectUuid, @Nullable String branchKey) {
+    return projectUuid + "-" + branchKey;
+  }
 
   private void resolveUnanalyzedLanguageCode(TelemetryData.Builder data, DbSession dbSession) {
     long numberOfUnanalyzedCMeasures = dbClient.liveMeasureDao().countProjectsHavingMeasure(dbSession, UNANALYZED_C_KEY);
@@ -216,6 +284,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
         .setVulnerabilities(metrics.getOrDefault("vulnerabilities", null))
         .setSecurityHotspots(metrics.getOrDefault("security_hotspots", null))
         .setTechnicalDebt(metrics.getOrDefault("sqale_index", null))
+        .setNcdId(ncdByProject.getOrDefault(projectUuid, instanceNcd).hashCode())
         .build();
       projectStatistics.add(stats);
     }
@@ -361,9 +430,6 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
     return configuration.get(property).isPresent();
   }
 
-  private boolean isScimEnabled() {
-    return this.internalProperties.read(SCIM_PROPERTY_ENABLED).map(Boolean::parseBoolean).orElse(false);
-  }
 
   private TelemetryData.ManagedInstanceInformation buildManagedInstanceInformation() {
     String provider = managedInstanceService.isInstanceExternallyManaged() ? managedInstanceService.getProviderName() : null;
diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java
index 2018cd1095c..0df240825c3 100644
--- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java
+++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java
@@ -48,6 +48,8 @@ import org.sonar.db.component.AnalysisPropertyDto;
 import org.sonar.db.component.ComponentDto;
 import org.sonar.db.component.SnapshotDto;
 import org.sonar.db.metric.MetricDto;
+import org.sonar.db.newcodeperiod.NewCodePeriodDto;
+import org.sonar.db.newcodeperiod.NewCodePeriodType;
 import org.sonar.db.qualitygate.QualityGateDto;
 import org.sonar.db.user.UserDbTester;
 import org.sonar.db.user.UserDto;
@@ -58,6 +60,8 @@ import org.sonar.server.property.InternalProperties;
 import org.sonar.server.property.MapInternalProperties;
 import org.sonar.server.qualitygate.QualityGateCaycChecker;
 import org.sonar.server.qualitygate.QualityGateFinder;
+import org.sonar.server.telemetry.TelemetryData.NewCodeDefinition;
+import org.sonar.server.telemetry.TelemetryData.ProjectStatistics;
 import org.sonar.updatecenter.common.Version;
 
 import static java.util.Arrays.asList;
@@ -107,9 +111,9 @@ public class TelemetryDataLoaderImplTest {
   private final ManagedInstanceService managedInstanceService = mock(ManagedInstanceService.class);
 
   private final TelemetryDataLoader communityUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
-      internalProperties, configuration, dockerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService);
+    internalProperties, configuration, dockerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService);
   private final TelemetryDataLoader commercialUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
-      internalProperties, configuration, dockerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService);
+    internalProperties, configuration, dockerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService);
 
   private QualityGateDto builtInDefaultQualityGate;
   private MetricDto bugsDto;
@@ -199,11 +203,22 @@ public class TelemetryDataLoaderImplTest {
     // link one project to a non-default QG
     db.qualityGates().associateProjectToQualityGate(db.components().getProjectDtoByMainBranch(project1), qualityGate1);
 
+    var branch1 = db.components().insertProjectBranch(project1, branchDto -> branchDto.setKey("reference"));
+    var branch2 = db.components().insertProjectBranch(project1, branchDto -> branchDto.setKey("custom"));
+
+    var ncd1 = db.newCodePeriods().insert(project1.uuid(), NewCodePeriodType.NUMBER_OF_DAYS, "30");
+    var ncd2 = db.newCodePeriods().insert(project1.uuid(), branch2.branchUuid(), NewCodePeriodType.REFERENCE_BRANCH, "reference");
+
+    var instanceNcdId = NewCodeDefinition.getInstanceDefault().hashCode();
+    var projectNcdId = new NewCodeDefinition(NewCodePeriodType.NUMBER_OF_DAYS.name(), "30", "project").hashCode();
+    var branchNcdId = new NewCodeDefinition(NewCodePeriodType.REFERENCE_BRANCH.name(), branch1.uuid(), "branch").hashCode();
+
     TelemetryData data = communityUnderTest.load();
     assertThat(data.getServerId()).isEqualTo(serverId);
     assertThat(data.getVersion()).isEqualTo(version);
     assertThat(data.getEdition()).contains(DEVELOPER);
     assertThat(data.getDefaultQualityGate()).isEqualTo(builtInDefaultQualityGate.getUuid());
+    assertThat(data.getNcdId()).isEqualTo(NewCodeDefinition.getInstanceDefault().hashCode());
     assertThat(data.getMessageSequenceNumber()).isOne();
     assertDatabaseMetadata(data.getDatabase());
     assertThat(data.getPlugins()).containsOnly(
@@ -226,13 +241,31 @@ public class TelemetryDataLoaderImplTest {
         tuple(project2.uuid(), "java", 180L, analysisDate),
         tuple(project2.uuid(), "js", 20L, analysisDate));
     assertThat(data.getProjectStatistics())
-      .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount, TelemetryData.ProjectStatistics::getQualityGate,
-        TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi, TelemetryData.ProjectStatistics::getDevopsPlatform,
-        TelemetryData.ProjectStatistics::getBugs, TelemetryData.ProjectStatistics::getVulnerabilities, TelemetryData.ProjectStatistics::getSecurityHotspots,
-        TelemetryData.ProjectStatistics::getDevelopmentCost, TelemetryData.ProjectStatistics::getTechnicalDebt)
+      .extracting(ProjectStatistics::getBranchCount, ProjectStatistics::getPullRequestCount, ProjectStatistics::getQualityGate,
+        ProjectStatistics::getScm, ProjectStatistics::getCi, ProjectStatistics::getDevopsPlatform,
+        ProjectStatistics::getBugs, ProjectStatistics::getVulnerabilities, ProjectStatistics::getSecurityHotspots,
+        ProjectStatistics::getDevelopmentCost, ProjectStatistics::getTechnicalDebt, ProjectStatistics::getNcdId)
+      .containsExactlyInAnyOrder(
+        tuple(3L, 0L, qualityGate1.getUuid(), "scm-1", "ci-1", "azure_devops_cloud", Optional.of(1L), Optional.of(1L), Optional.of(1L), Optional.of(50L), Optional.of(5L),
+          projectNcdId),
+        tuple(1L, 0L, builtInDefaultQualityGate.getUuid(), "scm-2", "ci-2", "github_cloud", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(),
+          Optional.empty(), instanceNcdId));
+
+    assertThat(data.getBranches())
+      .extracting(TelemetryData.Branch::branchUuid, TelemetryData.Branch::ncdId)
+      .containsExactlyInAnyOrder(
+        tuple(branch1.uuid(), projectNcdId),
+        tuple(branch2.uuid(), branchNcdId),
+        tuple(project1.uuid(), projectNcdId),
+        tuple(project2.uuid(), instanceNcdId));
+
+    assertThat(data.getNewCodeDefinitions())
+      .extracting(NewCodeDefinition::scope, NewCodeDefinition::type, NewCodeDefinition::value)
       .containsExactlyInAnyOrder(
-        tuple(1L, 0L, qualityGate1.getUuid(), "scm-1", "ci-1", "azure_devops_cloud", Optional.of(1L), Optional.of(1L), Optional.of(1L), Optional.of(50L), Optional.of(5L)),
-        tuple(1L, 0L, builtInDefaultQualityGate.getUuid(), "scm-2", "ci-2", "github_cloud", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()));
+        tuple("instance", NewCodePeriodType.PREVIOUS_VERSION.name(), ""),
+        tuple("project", NewCodePeriodType.NUMBER_OF_DAYS.name(), "30"),
+        tuple("branch", NewCodePeriodType.REFERENCE_BRANCH.name(), branch1.uuid()));
+
     assertThat(data.getQualityGates())
       .extracting(TelemetryData.QualityGate::uuid, TelemetryData.QualityGate::caycStatus)
       .containsExactlyInAnyOrder(
@@ -297,8 +330,8 @@ public class TelemetryDataLoaderImplTest {
         tuple(project.uuid(), "js", 50L),
         tuple(project.uuid(), "kotlin", 30L));
     assertThat(data.getProjectStatistics())
-      .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount,
-        TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
+      .extracting(ProjectStatistics::getBranchCount, ProjectStatistics::getPullRequestCount,
+        ProjectStatistics::getScm, ProjectStatistics::getCi)
       .containsExactlyInAnyOrder(
         tuple(2L, 0L, "undetected", "undetected"));
   }
@@ -436,7 +469,7 @@ public class TelemetryDataLoaderImplTest {
     db.components().insertPublicProject().getMainBranchComponent();
     TelemetryData data = communityUnderTest.load();
     assertThat(data.getProjectStatistics())
-      .extracting(TelemetryData.ProjectStatistics::getDevopsPlatform, TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
+      .extracting(ProjectStatistics::getDevopsPlatform, ProjectStatistics::getScm, ProjectStatistics::getCi)
       .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected"));
   }
 
@@ -460,7 +493,7 @@ public class TelemetryDataLoaderImplTest {
     db.qualityGates().setDefaultQualityGate(qualityGate);
     TelemetryData data = communityUnderTest.load();
     assertThat(data.getProjectStatistics())
-      .extracting(TelemetryData.ProjectStatistics::getQualityGate)
+      .extracting(ProjectStatistics::getQualityGate)
       .containsOnly(qualityGate.getUuid());
   }
 
@@ -489,7 +522,7 @@ public class TelemetryDataLoaderImplTest {
 
   @DataProvider
   public static Object[][] getManagedInstanceData() {
-    return new Object[][]{
+    return new Object[][] {
       {true, "scim"},
       {true, "github"},
       {true, "gitlab"},