diff options
Diffstat (limited to 'server/sonar-webserver-core/src')
3 files changed, 117 insertions, 17 deletions
diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java index 974219952ba..fe9ad60ea11 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java @@ -150,6 +150,7 @@ public class TelemetryDaemon extends AbstractStoppableScheduledExecutorServiceIm dataJsonWriter.writeTelemetryData(json, statistics); } telemetryClient.upload(jsonString.toString()); + dataLoader.reset(); } private boolean shouldUploadStatistics(long now) { diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java index 0d87f2b9e0b..03c441fe5b7 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java @@ -27,12 +27,14 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import javax.annotation.Nullable; import javax.inject.Inject; import org.sonar.api.config.Configuration; import org.sonar.api.platform.Server; @@ -46,11 +48,13 @@ import org.sonar.db.DbSession; import org.sonar.db.alm.setting.ALM; import org.sonar.db.alm.setting.ProjectAlmKeyAndProject; import org.sonar.db.component.AnalysisPropertyValuePerProject; +import org.sonar.db.component.BranchDto; import org.sonar.db.component.PrBranchAnalyzedLanguageCountByProjectDto; import org.sonar.db.component.SnapshotDto; import org.sonar.db.measure.LiveMeasureDto; import org.sonar.db.measure.ProjectLocDistributionDto; import org.sonar.db.metric.MetricDto; +import org.sonar.db.newcodeperiod.NewCodePeriodDto; import org.sonar.db.qualitygate.ProjectQgateAssociationDto; import org.sonar.db.qualitygate.QualityGateDto; import org.sonar.server.management.ManagedInstanceService; @@ -59,6 +63,7 @@ import org.sonar.server.property.InternalProperties; import org.sonar.server.qualitygate.QualityGateCaycChecker; import org.sonar.server.qualitygate.QualityGateFinder; import org.sonar.server.telemetry.TelemetryData.Database; +import org.sonar.server.telemetry.TelemetryData.NewCodeDefinition; import static java.util.Arrays.asList; import static java.util.Optional.ofNullable; @@ -77,6 +82,7 @@ import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETEC import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY; import static org.sonar.core.platform.EditionProvider.Edition.DATACENTER; import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH; import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY; import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY; import static org.sonar.server.telemetry.TelemetryDaemon.I_PROP_MESSAGE_SEQUENCE; @@ -103,6 +109,10 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { private final QualityGateCaycChecker qualityGateCaycChecker; private final QualityGateFinder qualityGateFinder; private final ManagedInstanceService managedInstanceService; + private final Set<NewCodeDefinition> newCodeDefinitions = new HashSet<>(); + private final Map<String, NewCodeDefinition> ncdByProject = new HashMap<>(); + private final Map<String, NewCodeDefinition> ncdByBranch = new HashMap<>(); + private NewCodeDefinition instanceNcd = NewCodeDefinition.getInstanceDefault(); @Inject public TelemetryDataLoaderImpl(Server server, DbClient dbClient, PluginRepository pluginRepository, @@ -143,7 +153,12 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { getVersion)); data.setPlugins(plugins); try (DbSession dbSession = dbClient.openSession(false)) { + var branchDtos = dbClient.branchDao().selectAllBranches(dbSession); + loadNewCodeDefinitions(dbSession, branchDtos); + data.setDatabase(loadDatabaseMetadata(dbSession)); + data.setNcdId(instanceNcd.hashCode()); + data.setNewCodeDefinitions(newCodeDefinitions); String defaultQualityGateUuid = qualityGateFinder.getDefault(dbSession).getUuid(); @@ -151,6 +166,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { resolveUnanalyzedLanguageCode(data, dbSession); resolveProjectStatistics(data, dbSession, defaultQualityGateUuid); resolveProjects(data, dbSession); + resolveBranches(data, branchDtos); resolveQualityGates(data, dbSession); resolveUsers(data, dbSession); } @@ -161,7 +177,6 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { installationDateProperty.ifPresent(s -> data.setInstallationDate(Long.valueOf(s))); Optional<String> installationVersionProperty = internalProperties.read(InternalProperties.INSTALLATION_VERSION); - return data .setInstallationVersion(installationVersionProperty.orElse(null)) .setInDocker(dockerSupport.isRunningInDocker()) @@ -169,6 +184,59 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { .build(); } + private void resolveBranches(TelemetryData.Builder data, List<BranchDto> branchDtos) { + var branches = branchDtos.stream() + .map(dto -> { + var projectNcd = ncdByProject.getOrDefault(dto.getProjectUuid(), instanceNcd); + var ncdId = ncdByBranch.getOrDefault(dto.getUuid(), projectNcd).hashCode(); + return new TelemetryData.Branch(dto.getProjectUuid(), dto.getUuid(), ncdId); + }) + .toList(); + data.setBranches(branches); + } + + @Override + public void reset() { + this.newCodeDefinitions.clear(); + this.ncdByBranch.clear(); + this.ncdByProject.clear(); + this.instanceNcd = NewCodeDefinition.getInstanceDefault(); + } + + private void loadNewCodeDefinitions(DbSession dbSession, List<BranchDto> branchDtos) { + var branchUuidByKey = branchDtos.stream().collect(Collectors.toMap(dto -> createBranchUniqueKey(dto.getProjectUuid(), dto.getBranchKey()), BranchDto::getUuid)); + List<NewCodePeriodDto> newCodePeriodDtos = dbClient.newCodePeriodDao().selectAll(dbSession); + NewCodeDefinition ncd; + boolean hasInstance = false; + for (var dto : newCodePeriodDtos) { + String projectUuid = dto.getProjectUuid(); + String branchUuid = dto.getBranchUuid(); + if (branchUuid == null && projectUuid == null) { + ncd = new NewCodeDefinition(dto.getType().name(), dto.getValue(), "instance"); + this.instanceNcd = ncd; + hasInstance = true; + } else if (projectUuid != null) { + var value = dto.getType() == REFERENCE_BRANCH ? branchUuidByKey.get(createBranchUniqueKey(projectUuid, dto.getValue())) : dto.getValue(); + if (branchUuid == null) { + ncd = new NewCodeDefinition(dto.getType().name(), value, "project"); + this.ncdByProject.put(projectUuid, ncd); + } else { + ncd = new NewCodeDefinition(dto.getType().name(), value, "branch"); + this.ncdByBranch.put(branchUuid, ncd); + } + } else { + throw new IllegalStateException(String.format("Error in loading telemetry data. New code definition for branch %s doesn't have a projectUuid", branchUuid)); + } + this.newCodeDefinitions.add(ncd); + } + if (!hasInstance) { + this.newCodeDefinitions.add(NewCodeDefinition.getInstanceDefault()); + } + } + + private static String createBranchUniqueKey(String projectUuid, @Nullable String branchKey) { + return projectUuid + "-" + branchKey; + } private void resolveUnanalyzedLanguageCode(TelemetryData.Builder data, DbSession dbSession) { long numberOfUnanalyzedCMeasures = dbClient.liveMeasureDao().countProjectsHavingMeasure(dbSession, UNANALYZED_C_KEY); @@ -216,6 +284,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { .setVulnerabilities(metrics.getOrDefault("vulnerabilities", null)) .setSecurityHotspots(metrics.getOrDefault("security_hotspots", null)) .setTechnicalDebt(metrics.getOrDefault("sqale_index", null)) + .setNcdId(ncdByProject.getOrDefault(projectUuid, instanceNcd).hashCode()) .build(); projectStatistics.add(stats); } @@ -361,9 +430,6 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { return configuration.get(property).isPresent(); } - private boolean isScimEnabled() { - return this.internalProperties.read(SCIM_PROPERTY_ENABLED).map(Boolean::parseBoolean).orElse(false); - } private TelemetryData.ManagedInstanceInformation buildManagedInstanceInformation() { String provider = managedInstanceService.isInstanceExternallyManaged() ? managedInstanceService.getProviderName() : null; diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java index 2018cd1095c..0df240825c3 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java @@ -48,6 +48,8 @@ import org.sonar.db.component.AnalysisPropertyDto; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.SnapshotDto; import org.sonar.db.metric.MetricDto; +import org.sonar.db.newcodeperiod.NewCodePeriodDto; +import org.sonar.db.newcodeperiod.NewCodePeriodType; import org.sonar.db.qualitygate.QualityGateDto; import org.sonar.db.user.UserDbTester; import org.sonar.db.user.UserDto; @@ -58,6 +60,8 @@ import org.sonar.server.property.InternalProperties; import org.sonar.server.property.MapInternalProperties; import org.sonar.server.qualitygate.QualityGateCaycChecker; import org.sonar.server.qualitygate.QualityGateFinder; +import org.sonar.server.telemetry.TelemetryData.NewCodeDefinition; +import org.sonar.server.telemetry.TelemetryData.ProjectStatistics; import org.sonar.updatecenter.common.Version; import static java.util.Arrays.asList; @@ -107,9 +111,9 @@ public class TelemetryDataLoaderImplTest { private final ManagedInstanceService managedInstanceService = mock(ManagedInstanceService.class); private final TelemetryDataLoader communityUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider, - internalProperties, configuration, dockerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService); + internalProperties, configuration, dockerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService); private final TelemetryDataLoader commercialUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider, - internalProperties, configuration, dockerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService); + internalProperties, configuration, dockerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService); private QualityGateDto builtInDefaultQualityGate; private MetricDto bugsDto; @@ -199,11 +203,22 @@ public class TelemetryDataLoaderImplTest { // link one project to a non-default QG db.qualityGates().associateProjectToQualityGate(db.components().getProjectDtoByMainBranch(project1), qualityGate1); + var branch1 = db.components().insertProjectBranch(project1, branchDto -> branchDto.setKey("reference")); + var branch2 = db.components().insertProjectBranch(project1, branchDto -> branchDto.setKey("custom")); + + var ncd1 = db.newCodePeriods().insert(project1.uuid(), NewCodePeriodType.NUMBER_OF_DAYS, "30"); + var ncd2 = db.newCodePeriods().insert(project1.uuid(), branch2.branchUuid(), NewCodePeriodType.REFERENCE_BRANCH, "reference"); + + var instanceNcdId = NewCodeDefinition.getInstanceDefault().hashCode(); + var projectNcdId = new NewCodeDefinition(NewCodePeriodType.NUMBER_OF_DAYS.name(), "30", "project").hashCode(); + var branchNcdId = new NewCodeDefinition(NewCodePeriodType.REFERENCE_BRANCH.name(), branch1.uuid(), "branch").hashCode(); + TelemetryData data = communityUnderTest.load(); assertThat(data.getServerId()).isEqualTo(serverId); assertThat(data.getVersion()).isEqualTo(version); assertThat(data.getEdition()).contains(DEVELOPER); assertThat(data.getDefaultQualityGate()).isEqualTo(builtInDefaultQualityGate.getUuid()); + assertThat(data.getNcdId()).isEqualTo(NewCodeDefinition.getInstanceDefault().hashCode()); assertThat(data.getMessageSequenceNumber()).isOne(); assertDatabaseMetadata(data.getDatabase()); assertThat(data.getPlugins()).containsOnly( @@ -226,13 +241,31 @@ public class TelemetryDataLoaderImplTest { tuple(project2.uuid(), "java", 180L, analysisDate), tuple(project2.uuid(), "js", 20L, analysisDate)); assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount, TelemetryData.ProjectStatistics::getQualityGate, - TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi, TelemetryData.ProjectStatistics::getDevopsPlatform, - TelemetryData.ProjectStatistics::getBugs, TelemetryData.ProjectStatistics::getVulnerabilities, TelemetryData.ProjectStatistics::getSecurityHotspots, - TelemetryData.ProjectStatistics::getDevelopmentCost, TelemetryData.ProjectStatistics::getTechnicalDebt) + .extracting(ProjectStatistics::getBranchCount, ProjectStatistics::getPullRequestCount, ProjectStatistics::getQualityGate, + ProjectStatistics::getScm, ProjectStatistics::getCi, ProjectStatistics::getDevopsPlatform, + ProjectStatistics::getBugs, ProjectStatistics::getVulnerabilities, ProjectStatistics::getSecurityHotspots, + ProjectStatistics::getDevelopmentCost, ProjectStatistics::getTechnicalDebt, ProjectStatistics::getNcdId) + .containsExactlyInAnyOrder( + tuple(3L, 0L, qualityGate1.getUuid(), "scm-1", "ci-1", "azure_devops_cloud", Optional.of(1L), Optional.of(1L), Optional.of(1L), Optional.of(50L), Optional.of(5L), + projectNcdId), + tuple(1L, 0L, builtInDefaultQualityGate.getUuid(), "scm-2", "ci-2", "github_cloud", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), instanceNcdId)); + + assertThat(data.getBranches()) + .extracting(TelemetryData.Branch::branchUuid, TelemetryData.Branch::ncdId) + .containsExactlyInAnyOrder( + tuple(branch1.uuid(), projectNcdId), + tuple(branch2.uuid(), branchNcdId), + tuple(project1.uuid(), projectNcdId), + tuple(project2.uuid(), instanceNcdId)); + + assertThat(data.getNewCodeDefinitions()) + .extracting(NewCodeDefinition::scope, NewCodeDefinition::type, NewCodeDefinition::value) .containsExactlyInAnyOrder( - tuple(1L, 0L, qualityGate1.getUuid(), "scm-1", "ci-1", "azure_devops_cloud", Optional.of(1L), Optional.of(1L), Optional.of(1L), Optional.of(50L), Optional.of(5L)), - tuple(1L, 0L, builtInDefaultQualityGate.getUuid(), "scm-2", "ci-2", "github_cloud", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())); + tuple("instance", NewCodePeriodType.PREVIOUS_VERSION.name(), ""), + tuple("project", NewCodePeriodType.NUMBER_OF_DAYS.name(), "30"), + tuple("branch", NewCodePeriodType.REFERENCE_BRANCH.name(), branch1.uuid())); + assertThat(data.getQualityGates()) .extracting(TelemetryData.QualityGate::uuid, TelemetryData.QualityGate::caycStatus) .containsExactlyInAnyOrder( @@ -297,8 +330,8 @@ public class TelemetryDataLoaderImplTest { tuple(project.uuid(), "js", 50L), tuple(project.uuid(), "kotlin", 30L)); assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount, - TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi) + .extracting(ProjectStatistics::getBranchCount, ProjectStatistics::getPullRequestCount, + ProjectStatistics::getScm, ProjectStatistics::getCi) .containsExactlyInAnyOrder( tuple(2L, 0L, "undetected", "undetected")); } @@ -436,7 +469,7 @@ public class TelemetryDataLoaderImplTest { db.components().insertPublicProject().getMainBranchComponent(); TelemetryData data = communityUnderTest.load(); assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::getDevopsPlatform, TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi) + .extracting(ProjectStatistics::getDevopsPlatform, ProjectStatistics::getScm, ProjectStatistics::getCi) .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected")); } @@ -460,7 +493,7 @@ public class TelemetryDataLoaderImplTest { db.qualityGates().setDefaultQualityGate(qualityGate); TelemetryData data = communityUnderTest.load(); assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::getQualityGate) + .extracting(ProjectStatistics::getQualityGate) .containsOnly(qualityGate.getUuid()); } @@ -489,7 +522,7 @@ public class TelemetryDataLoaderImplTest { @DataProvider public static Object[][] getManagedInstanceData() { - return new Object[][]{ + return new Object[][] { {true, "scim"}, {true, "github"}, {true, "gitlab"}, |