diff options
author | Alain Kermis <alain.kermis@sonarsource.com> | 2023-01-25 15:20:50 +0100 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2023-02-03 14:41:20 +0000 |
commit | cf55c2180b2217738e29a3ebe7236a9c5b572379 (patch) | |
tree | 7ff16884ae6d3e64effe5cd2684f7fa3a40bdb75 | |
parent | 2d1ab592f000d85e8b41e5c00a76ac47ed0ee48d (diff) | |
download | sonarqube-cf55c2180b2217738e29a3ebe7236a9c5b572379.tar.gz sonarqube-cf55c2180b2217738e29a3ebe7236a9c5b572379.zip |
SONAR-18219 Add telemetry fields for CaYC
9 files changed, 372 insertions, 57 deletions
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java index 4534c88b745..7e5547fc292 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java @@ -42,14 +42,18 @@ public class LiveMeasureDao implements Dao { this.system2 = system2; } - public List<LiveMeasureDto> selectByComponentUuidsAndMetricUuids(DbSession dbSession, Collection<String> largeComponentUuids, Collection<String> metricUuis) { - if (largeComponentUuids.isEmpty() || metricUuis.isEmpty()) { + public List<LiveMeasureDto> selectByComponentUuidsAndMetricUuids(DbSession dbSession, Collection<String> largeComponentUuids, Collection<String> metricUuids) { + if (largeComponentUuids.isEmpty() || metricUuids.isEmpty()) { return Collections.emptyList(); } return executeLargeInputs( largeComponentUuids, - componentUuids -> mapper(dbSession).selectByComponentUuidsAndMetricUuids(componentUuids, metricUuis)); + componentUuids -> mapper(dbSession).selectByComponentUuidsAndMetricUuids(componentUuids, metricUuids)); + } + + public List<LiveMeasureDto> selectForProjectsByMetricUuids(DbSession dbSession, Collection<String> metricUuids) { + return mapper(dbSession).selectForProjectsByMetricUuids(metricUuids); } public void scrollSelectByComponentUuidAndMetricKeys(DbSession dbSession, String componentUuid, Collection<String> metricKeys, ResultHandler<LiveMeasureDto> handler) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java index c7bd47c479a..9d2765d89e3 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java @@ -31,6 +31,9 @@ public interface LiveMeasureMapper { @Param("componentUuids") Collection<String> componentUuids, @Param("metricUuids") Collection<String> metricUuids); + List<LiveMeasureDto> selectForProjectsByMetricUuids( + @Param("metricUuids") Collection<String> metricUuids); + List<LiveMeasureDto> selectByComponentUuidsAndMetricKeys( @Param("componentUuids") Collection<String> componentUuids, @Param("metricKeys") Collection<String> metricKeys); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml index efdb50c965c..03d57e505bc 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml @@ -23,6 +23,16 @@ </foreach> </select> + <select id="selectForProjectsByMetricUuids" parameterType="map" resultType="org.sonar.db.measure.LiveMeasureDto"> + select <include refid="columns"/> from live_measures lm + where + lm.metric_uuid in <foreach item="metricUuid" collection="metricUuids" open="(" separator="," + close=")">#{metricUuid, jdbcType=VARCHAR}</foreach> + and component_uuid in (SELECT + p.uuid as uuid + FROM projects p) + </select> + <select id="selectByComponentUuidsAndMetricKeys" parameterType="map" resultType="org.sonar.db.measure.LiveMeasureDto"> select <include refid="columns"/> from live_measures lm inner join metrics m on m.uuid = lm.metric_uuid diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java index 4904b752d54..aba81a1f63e 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java @@ -142,6 +142,33 @@ public class LiveMeasureDaoTest { } @Test + public void selectForProjectsByMetricUuids() { + MetricDto metric = db.measures().insertMetric(); + MetricDto metric2 = db.measures().insertMetric(); + ComponentDto project = db.components().insertPrivateProject(); + ComponentDto project2 = db.components().insertPrivateProject(); + underTest.insert(db.getSession(), newLiveMeasure(project, metric).setValue(3.14).setData((String) null)); + underTest.insert(db.getSession(), newLiveMeasure(project, metric2).setValue(4.54).setData((String) null)); + underTest.insert(db.getSession(), newLiveMeasure(project2, metric).setValue(99.99).setData((String) null)); + + List<LiveMeasureDto> selected = underTest.selectForProjectsByMetricUuids(db.getSession(), List.of(metric.getUuid(), metric2.getUuid())); + assertThat(selected) + .extracting(LiveMeasureDto::getComponentUuid, LiveMeasureDto::getProjectUuid, LiveMeasureDto::getMetricUuid, LiveMeasureDto::getValue, LiveMeasureDto::getDataAsString) + .containsExactlyInAnyOrder( + tuple(project.uuid(), project.uuid(), metric.getUuid(), 3.14, null), + tuple(project.uuid(), project.uuid(), metric2.getUuid(), 4.54, null), + tuple(project2.uuid(), project2.uuid(), metric.getUuid(), 99.99, null)); + } + + @Test + public void selectForProjectsByMetricUuids_whenMetricDoesNotMatch_shouldReturnEmptyList() { + ComponentDto project = db.components().insertPrivateProject(); + underTest.insert(db.getSession(), newLiveMeasure(project, metric).setValue(3.14).setData((String) null)); + List<LiveMeasureDto> selected = underTest.selectForProjectsByMetricUuids(db.getSession(), singletonList("_other_")); + assertThat(selected).isEmpty(); + } + + @Test public void selectByComponentUuidAndMetricKey() { LiveMeasureDto measure = newLiveMeasure().setMetricUuid(metric.getUuid()); underTest.insert(db.getSession(), measure); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java index 6daa1256f8a..9ab24fcde4a 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java @@ -280,9 +280,163 @@ public class TelemetryData { record Project(String projectUuid, Long lastAnalysis, String language, Long loc) { } - record ProjectStatistics(String projectUuid, Long branchCount, Long pullRequestCount, String qualityGate, String scm, String ci, String devopsPlatform) { + record QualityGate(String uuid, boolean isCaycCompliant) { } - record QualityGate(String uuid, boolean isCaycCompliant) { + public static class ProjectStatistics { + private final String projectUuid; + private final Long branchCount; + private final Long pullRequestCount; + private final String qualityGate; + private final String scm; + private final String ci; + private final String devopsPlatform; + private final Long bugs; + private final Long vulnerabilities; + private final Long securityHotspots; + private final Double technicalDebt; + private final Double developmentCost; + + ProjectStatistics(Builder builder) { + this.projectUuid = builder.projectUuid; + this.branchCount = builder.branchCount; + this.pullRequestCount = builder.pullRequestCount; + this.qualityGate = builder.qualityGate; + this.scm = builder.scm; + this.ci = builder.ci; + this.devopsPlatform = builder.devopsPlatform; + this.bugs = builder.bugs; + this.vulnerabilities = builder.vulnerabilities; + this.securityHotspots = builder.securityHotspots; + this.technicalDebt = builder.technicalDebt; + this.developmentCost = builder.developmentCost; + } + + public String getProjectUuid() { + return projectUuid; + } + + public Long getBranchCount() { + return branchCount; + } + + public Long getPullRequestCount() { + return pullRequestCount; + } + + public String getQualityGate() { + return qualityGate; + } + + public String getScm() { + return scm; + } + + public String getCi() { + return ci; + } + + public String getDevopsPlatform() { + return devopsPlatform; + } + + public Optional<Long> getBugs() { + return Optional.ofNullable(bugs); + } + + public Optional<Long> getVulnerabilities() { + return Optional.ofNullable(vulnerabilities); + } + + public Optional<Long> getSecurityHotspots() { + return Optional.ofNullable(securityHotspots); + } + + public Optional<Double> getTechnicalDebt() { + return Optional.ofNullable(technicalDebt); + } + + public Optional<Double> getDevelopmentCost() { + return Optional.ofNullable(developmentCost); + } + + static class Builder { + private String projectUuid; + private Long branchCount; + private Long pullRequestCount; + private String qualityGate; + private String scm; + private String ci; + private String devopsPlatform; + private Long bugs; + private Long vulnerabilities; + private Long securityHotspots; + private Double technicalDebt; + private Double developmentCost; + + public Builder setProjectUuid(String projectUuid) { + this.projectUuid = projectUuid; + return this; + } + + public Builder setBranchCount(Long branchCount) { + this.branchCount = branchCount; + return this; + } + + public Builder setPRCount(Long pullRequestCount) { + this.pullRequestCount = pullRequestCount; + return this; + } + + public Builder setQG(String qualityGate) { + this.qualityGate = qualityGate; + return this; + } + + public Builder setScm(String scm) { + this.scm = scm; + return this; + } + + public Builder setCi(String ci) { + this.ci = ci; + return this; + } + + public Builder setDevops(String devopsPlatform) { + this.devopsPlatform = devopsPlatform; + return this; + } + + public Builder setBugs(@Nullable Number bugs) { + this.bugs = bugs != null ? bugs.longValue() : null; + return this; + } + + public Builder setVulnerabilities(@Nullable Number vulnerabilities) { + this.vulnerabilities = vulnerabilities != null ? vulnerabilities.longValue() : null; + return this; + } + + public Builder setSecurityHotspots(@Nullable Number securityHotspots) { + this.securityHotspots = securityHotspots != null ? securityHotspots.longValue() : null; + return this; + } + + public Builder setTechnicalDebt(@Nullable Number technicalDebt) { + this.technicalDebt = technicalDebt != null ? technicalDebt.doubleValue() : null; + return this; + } + + public Builder setDevelopmentCost(@Nullable Number developmentCost) { + this.developmentCost = developmentCost != null ? developmentCost.doubleValue() : null; + return this; + } + + public ProjectStatistics build() { + return new ProjectStatistics(this); + } + } } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java index 116612e8b02..57171896fdb 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java @@ -149,13 +149,18 @@ public class TelemetryDataJsonWriter { json.beginArray(); statistics.getProjectStatistics().forEach(project -> { json.beginObject(); - json.prop("projectUuid", project.projectUuid()); - json.prop("branchCount", project.branchCount()); - json.prop("pullRequestCount", project.pullRequestCount()); - json.prop("qualityGate", project.qualityGate()); - json.prop("scm", project.scm()); - json.prop("ci", project.ci()); - json.prop("devopsPlatform", project.devopsPlatform()); + json.prop("projectUuid", project.getProjectUuid()); + json.prop("branchCount", project.getBranchCount()); + json.prop("pullRequestCount", project.getPullRequestCount()); + json.prop("qualityGate", project.getQualityGate()); + json.prop("scm", project.getScm()); + json.prop("ci", project.getCi()); + json.prop("devopsPlatform", project.getDevopsPlatform()); + project.getBugs().ifPresent(bugs -> json.prop("bugs", bugs)); + project.getVulnerabilities().ifPresent(vulnerabilities -> json.prop("vulnerabilities", vulnerabilities)); + project.getSecurityHotspots().ifPresent(securityHotspots -> json.prop("securityHotspots", securityHotspots)); + project.getTechnicalDebt().ifPresent(technicalDebt -> json.prop("technicalDebt", technicalDebt)); + project.getDevelopmentCost().ifPresent(developmentCost -> json.prop("developmentCost", developmentCost)); json.endObject(); }); json.endArray(); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java index d3dc46faa15..67814020702 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java @@ -366,7 +366,7 @@ public class TelemetryDataJsonWriterTest { @Test public void writes_all_projects_stats_with_analyzed_languages() { TelemetryData data = telemetryBuilder() - .setProjectStatistics(attachProjectStats()) + .setProjectStatistics(attachProjectStatsWithMetrics()) .build(); String json = writeTelemetryData(data); @@ -378,28 +378,43 @@ public class TelemetryDataJsonWriterTest { "projectUuid": "uuid-0", "branchCount": 2, "pullRequestCount": 2, - "qualityGate": "qg-0" + "qualityGate": "qg-0", "scm": "scm-0", "ci": "ci-0", - "devopsPlatform": "devops-0" + "devopsPlatform": "devops-0", + "bugs": 2, + "vulnerabilities": 3, + "securityHotspots": 4, + "technicalDebt": 60.0, + "developmentCost": 30.0 }, { "projectUuid": "uuid-1", "branchCount": 4, "pullRequestCount": 4, - "qualityGate": "qg-1" + "qualityGate": "qg-1", "scm": "scm-1", "ci": "ci-1", - "devopsPlatform": "devops-1" + "devopsPlatform": "devops-1", + "bugs": 4, + "vulnerabilities": 6, + "securityHotspots": 8, + "technicalDebt": 120.0, + "developmentCost": 60.0 }, { "projectUuid": "uuid-2", "branchCount": 6, "pullRequestCount": 6, - "qualityGate": "qg-2" + "qualityGate": "qg-2", "scm": "scm-2", "ci": "ci-2", - "devopsPlatform": "devops-2" + "devopsPlatform": "devops-2", + "bugs": 6, + "vulnerabilities": 9, + "securityHotspots": 12, + "technicalDebt": 180.0, + "developmentCost": 90.0 } ] } @@ -418,6 +433,15 @@ public class TelemetryDataJsonWriterTest { } @Test + public void writes_all_projects_stats_without_missing_metrics() { + TelemetryData data = telemetryBuilder() + .setProjectStatistics(attachProjectStats()) + .build(); + String json = writeTelemetryData(data); + assertThat(json).doesNotContain("bugs", "vulnerabilities", "securityHotspots", "technicalDebt", "developmentCost"); + } + + @Test public void writes_all_quality_gates() { TelemetryData data = telemetryBuilder() .setQualityGates(attachQualityGates()) @@ -466,9 +490,31 @@ public class TelemetryDataJsonWriterTest { return IntStream.range(0, 3).mapToObj(i -> new TelemetryData.Project("uuid-" + i, 1L, "lang-" + i, (i + 1L) * 2L)).collect(Collectors.toList()); } - private List<TelemetryData.ProjectStatistics> attachProjectStats() { - return IntStream.range(0, 3).mapToObj(i -> new TelemetryData.ProjectStatistics("uuid-" + i, (i + 1L) * 2L, (i + 1L) * 2L, "qg-" + i, "scm-" + i, "ci-" + i, "devops-" + i)) - .collect(Collectors.toList()); + private static List<TelemetryData.ProjectStatistics> attachProjectStatsWithMetrics() { + return IntStream.range(0, 3).mapToObj(i -> getProjectStatisticsWithMetricBuilder(i).build()).toList(); + } + + private static List<TelemetryData.ProjectStatistics> attachProjectStats() { + return IntStream.range(0, 3).mapToObj(i -> getProjectStatisticsBuilder(i).build()).toList(); + } + + private static TelemetryData.ProjectStatistics.Builder getProjectStatisticsBuilder(int i) { + return new TelemetryData.ProjectStatistics.Builder() + .setProjectUuid("uuid-" + i) + .setBranchCount((i + 1L) * 2L) + .setPRCount((i + 1L) * 2L) + .setQG("qg-" + i).setCi("ci-" + i) + .setScm("scm-" + i) + .setDevops("devops-" + i); + } + + private static TelemetryData.ProjectStatistics.Builder getProjectStatisticsWithMetricBuilder(int i) { + return getProjectStatisticsBuilder(i) + .setBugs((i + 1L) * 2) + .setVulnerabilities((i + 1L) * 3) + .setSecurityHotspots((i + 1L) * 4) + .setDevelopmentCost((i + 1L) * 30d) + .setTechnicalDebt((i + 1L) * 60d); } private List<TelemetryData.QualityGate> attachQualityGates() { @@ -479,7 +525,7 @@ public class TelemetryDataJsonWriterTest { @DataProvider public static Object[][] allEditions() { return Arrays.stream(EditionProvider.Edition.values()) - .map(t -> new Object[] {t}) + .map(t -> new Object[]{t}) .toArray(Object[][]::new); } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java index 9e79bb1dabf..b82f39156b3 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java @@ -24,6 +24,8 @@ import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -44,7 +46,9 @@ import org.sonar.db.alm.setting.ALM; import org.sonar.db.alm.setting.ProjectAlmKeyAndProject; import org.sonar.db.component.AnalysisPropertyValuePerProject; import org.sonar.db.component.PrBranchAnalyzedLanguageCountByProjectDto; +import org.sonar.db.measure.LiveMeasureDto; import org.sonar.db.measure.ProjectMeasureDto; +import org.sonar.db.metric.MetricDto; import org.sonar.db.qualitygate.ProjectQgateAssociationDto; import org.sonar.db.qualitygate.QualityGateDto; import org.sonar.server.platform.DockerSupport; @@ -55,8 +59,15 @@ import org.sonar.server.telemetry.TelemetryData.Database; import static java.util.Arrays.asList; import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toMap; import static org.sonar.api.internal.apachecommons.lang.StringUtils.startsWithIgnoreCase; +import static org.sonar.api.measures.CoreMetrics.BUGS_KEY; +import static org.sonar.api.measures.CoreMetrics.DEVELOPMENT_COST_KEY; import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_KEY; +import static org.sonar.api.measures.CoreMetrics.TECHNICAL_DEBT_KEY; +import static org.sonar.api.measures.CoreMetrics.VULNERABILITIES_KEY; import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI; import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM; import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY; @@ -121,7 +132,8 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { data.setVersion(server.getVersion()); data.setEdition(editionProvider.get().orElse(null)); Function<PluginInfo, String> getVersion = plugin -> plugin.getVersion() == null ? "undefined" : plugin.getVersion().getName(); - Map<String, String> plugins = pluginRepository.getPluginInfos().stream().collect(MoreCollectors.uniqueIndex(PluginInfo::getKey, getVersion)); + Map<String, String> plugins = pluginRepository.getPluginInfos().stream().collect(MoreCollectors.uniqueIndex(PluginInfo::getKey, + getVersion)); data.setPlugins(plugins); try (DbSession dbSession = dbClient.openSession(false)) { data.setDatabase(loadDatabaseMetadata(dbSession)); @@ -169,34 +181,46 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { Map<String, String> scmByProject = getAnalysisPropertyByProject(dbSession, SONAR_ANALYSIS_DETECTEDSCM); Map<String, String> ciByProject = getAnalysisPropertyByProject(dbSession, SONAR_ANALYSIS_DETECTEDCI); Map<String, ProjectAlmKeyAndProject> almAndUrlByProject = getAlmAndUrlByProject(dbSession); - Map<String, PrBranchAnalyzedLanguageCountByProjectDto> prAndBranchCountByProjects = dbClient.branchDao().countPrBranchAnalyzedLanguageByProjectUuid(dbSession) - .stream().collect(Collectors.toMap(PrBranchAnalyzedLanguageCountByProjectDto::getProjectUuid, Function.identity())); - - - Map<String, String> projectQgatesMap = getProjectQgatesMap(dbSession); + Map<String, PrBranchAnalyzedLanguageCountByProjectDto> prAndBranchCountByProject = + dbClient.branchDao().countPrBranchAnalyzedLanguageByProjectUuid(dbSession) + .stream().collect(toMap(PrBranchAnalyzedLanguageCountByProjectDto::getProjectUuid, Function.identity())); + Map<String, String> qgatesByProject = getProjectQgatesMap(dbSession); + Map<String, Map<String, Number>> metricsByProject = + getProjectMetricsByMetricKeys(dbSession, TECHNICAL_DEBT_KEY, DEVELOPMENT_COST_KEY, SECURITY_HOTSPOTS_KEY, VULNERABILITIES_KEY, + BUGS_KEY); List<TelemetryData.ProjectStatistics> projectStatistics = new ArrayList<>(); for (String projectUuid : projectUuids) { - Optional<PrBranchAnalyzedLanguageCountByProjectDto> counts = ofNullable(prAndBranchCountByProjects.get(projectUuid)); - - Long branchCount = counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getBranch).orElse(0L); - Long pullRequestCount = counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getPullRequest).orElse(0L); - String qualityGate = projectQgatesMap.getOrDefault(projectUuid, defaultQualityGateUuid); - String scm = Optional.ofNullable(scmByProject.get(projectUuid)).orElse(UNDETECTED); - String ci = Optional.ofNullable(ciByProject.get(projectUuid)).orElse(UNDETECTED); - String devopsPlatform = null; - if (almAndUrlByProject.containsKey(projectUuid)) { - ProjectAlmKeyAndProject projectAlmKeyAndProject = almAndUrlByProject.get(projectUuid); - devopsPlatform = getAlmName(projectAlmKeyAndProject.getAlmId(), projectAlmKeyAndProject.getUrl()); - } - devopsPlatform = Optional.ofNullable(devopsPlatform).orElse(UNDETECTED); - - projectStatistics.add( - new TelemetryData.ProjectStatistics(projectUuid, branchCount, pullRequestCount, qualityGate, scm, ci, devopsPlatform)); + Map<String, Number> metrics = metricsByProject.getOrDefault(projectUuid, Collections.emptyMap()); + Optional<PrBranchAnalyzedLanguageCountByProjectDto> counts = ofNullable(prAndBranchCountByProject.get(projectUuid)); + + TelemetryData.ProjectStatistics stats = new TelemetryData.ProjectStatistics.Builder() + .setProjectUuid(projectUuid) + .setBranchCount(counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getBranch).orElse(0L)) + .setPRCount(counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getPullRequest).orElse(0L)) + .setQG(qgatesByProject.getOrDefault(projectUuid, defaultQualityGateUuid)) + .setScm(Optional.ofNullable(scmByProject.get(projectUuid)).orElse(UNDETECTED)) + .setCi(Optional.ofNullable(ciByProject.get(projectUuid)).orElse(UNDETECTED)) + .setDevops(resolveDevopsPlatform(almAndUrlByProject, projectUuid)) + .setBugs(metrics.getOrDefault("bugs", null)) + .setDevelopmentCost(metrics.getOrDefault("development_cost", null)) + .setVulnerabilities(metrics.getOrDefault("vulnerabilities", null)) + .setSecurityHotspots(metrics.getOrDefault("security_hotspots", null)) + .setTechnicalDebt(metrics.getOrDefault("sqale_index", null)) + .build(); + projectStatistics.add(stats); } data.setProjectStatistics(projectStatistics); } + private static String resolveDevopsPlatform(Map<String, ProjectAlmKeyAndProject> almAndUrlByProject, String projectUuid) { + if (almAndUrlByProject.containsKey(projectUuid)) { + ProjectAlmKeyAndProject projectAlmKeyAndProject = almAndUrlByProject.get(projectUuid); + return getAlmName(projectAlmKeyAndProject.getAlmId(), projectAlmKeyAndProject.getUrl()); + } + return UNDETECTED; + } + private void resolveProjects(TelemetryData.Builder data, DbSession dbSession) { List<ProjectMeasureDto> measures = dbClient.measureDao().selectLastMeasureForAllProjects(dbSession, NCLOC_LANGUAGE_DISTRIBUTION_KEY); List<TelemetryData.Project> projects = new ArrayList<>(); @@ -216,7 +240,8 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { Collection<QualityGateDto> qualityGateDtos = dbClient.qualityGateDao().selectAll(dbSession); for (QualityGateDto qualityGateDto : qualityGateDtos) { qualityGates.add( - new TelemetryData.QualityGate(qualityGateDto.getUuid(), qualityGateCaycChecker.checkCaycCompliant(dbSession, qualityGateDto.getUuid())) + new TelemetryData.QualityGate(qualityGateDto.getUuid(), qualityGateCaycChecker.checkCaycCompliant(dbSession, + qualityGateDto.getUuid())) ); } @@ -237,12 +262,12 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { return dbClient.analysisPropertiesDao() .selectAnalysisPropertyValueInLastAnalysisPerProject(dbSession, analysisPropertyKey) .stream() - .collect(Collectors.toMap(AnalysisPropertyValuePerProject::getProjectUuid, AnalysisPropertyValuePerProject::getPropertyValue)); + .collect(toMap(AnalysisPropertyValuePerProject::getProjectUuid, AnalysisPropertyValuePerProject::getPropertyValue)); } private Map<String, ProjectAlmKeyAndProject> getAlmAndUrlByProject(DbSession dbSession) { List<ProjectAlmKeyAndProject> projectAlmKeyAndProjects = dbClient.projectAlmSettingDao().selectAlmTypeAndUrlByProject(dbSession); - return projectAlmKeyAndProjects.stream().collect(Collectors.toMap(ProjectAlmKeyAndProject::getProjectUuid, Function.identity())); + return projectAlmKeyAndProjects.stream().collect(toMap(ProjectAlmKeyAndProject::getProjectUuid, Function.identity())); } private static String getAlmName(String alm, String url) { @@ -268,7 +293,25 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { private Map<String, String> getProjectQgatesMap(DbSession dbSession) { return dbClient.projectQgateAssociationDao().selectAll(dbSession) .stream() - .collect(Collectors.toMap(ProjectQgateAssociationDto::getUuid, p -> Optional.ofNullable(p.getGateUuid()).orElse(""))); + .collect(toMap(ProjectQgateAssociationDto::getUuid, p -> Optional.ofNullable(p.getGateUuid()).orElse(""))); + } + + private Map<String, Map<String, Number>> getProjectMetricsByMetricKeys(DbSession dbSession, String... metricKeys) { + Map<String, String> metricNamesByUuid = dbClient.metricDao().selectByKeys(dbSession, asList(metricKeys)) + .stream() + .collect(toMap(MetricDto::getUuid, MetricDto::getKey)); + + // metrics can be empty for un-analyzed projects + if (metricNamesByUuid.isEmpty()) { + return Collections.emptyMap(); + } + + return dbClient.liveMeasureDao().selectForProjectsByMetricUuids(dbSession, metricNamesByUuid.keySet()) + .stream() + .collect(groupingBy(LiveMeasureDto::getProjectUuid, + toMap(lmDto -> metricNamesByUuid.get(lmDto.getMetricUuid()), + lmDto -> Optional.ofNullable(lmDto.getValue()).orElseGet(() -> Double.valueOf(lmDto.getTextValue())), + (oldValue, newValue) -> newValue, HashMap::new))); } private static boolean checkIfCloudAlm(String almRaw, String alm, String url, String cloudUrl) { diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java index 16f742cfa19..95b472f58ac 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java @@ -66,10 +66,15 @@ import static org.assertj.core.groups.Tuple.tuple; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import static org.sonar.api.measures.CoreMetrics.BUGS_KEY; import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY; +import static org.sonar.api.measures.CoreMetrics.DEVELOPMENT_COST_KEY; import static org.sonar.api.measures.CoreMetrics.LINES_KEY; import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_KEY; +import static org.sonar.api.measures.CoreMetrics.TECHNICAL_DEBT_KEY; +import static org.sonar.api.measures.CoreMetrics.VULNERABILITIES_KEY; import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI; import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM; import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY; @@ -103,12 +108,23 @@ public class TelemetryDataLoaderImplTest { internalProperties, configuration, dockerSupport, qualityGateCaycChecker, qualityGateFinder); private QualityGateDto builtInDefaultQualityGate; + private MetricDto bugsDto; + private MetricDto vulnerabilitiesDto; + private MetricDto securityHotspotsDto; + private MetricDto technicalDebtDto; + private MetricDto developmentCostDto; @Before public void setUpBuiltInQualityGate() { String builtInQgName = "Sonar Way"; builtInDefaultQualityGate = db.qualityGates().insertQualityGate(qg -> qg.setName(builtInQgName).setBuiltIn(true)); db.qualityGates().setDefaultQualityGate(builtInDefaultQualityGate); + + bugsDto = db.measures().insertMetric(m -> m.setKey(BUGS_KEY)); + vulnerabilitiesDto = db.measures().insertMetric(m -> m.setKey(VULNERABILITIES_KEY)); + securityHotspotsDto = db.measures().insertMetric(m -> m.setKey(SECURITY_HOTSPOTS_KEY)); + technicalDebtDto = db.measures().insertMetric(m -> m.setKey(TECHNICAL_DEBT_KEY)); + developmentCostDto = db.measures().insertMetric(m -> m.setKey(DEVELOPMENT_COST_KEY)); } @Test @@ -141,6 +157,11 @@ public class TelemetryDataLoaderImplTest { db.measures().insertLiveMeasure(project1, ncloc, m -> m.setValue(110d)); db.measures().insertLiveMeasure(project1, coverage, m -> m.setValue(80d)); db.measures().insertLiveMeasure(project1, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10")); + db.measures().insertLiveMeasure(project1, bugsDto, m -> m.setValue(1d)); + db.measures().insertLiveMeasure(project1, vulnerabilitiesDto, m -> m.setValue(1d).setData((String) null)); + db.measures().insertLiveMeasure(project1, securityHotspotsDto, m -> m.setValue(1d).setData((String) null)); + db.measures().insertLiveMeasure(project1, developmentCostDto, m -> m.setData("50").setValue(null)); + db.measures().insertLiveMeasure(project1, technicalDebtDto, m -> m.setValue(5d).setData((String) null)); ComponentDto project2 = db.components().insertPrivateProject(); db.measures().insertLiveMeasure(project2, lines, m -> m.setValue(200d)); @@ -200,11 +221,13 @@ public class TelemetryDataLoaderImplTest { tuple(project2.uuid(), "java", 180L, analysisDate), tuple(project2.uuid(), "js", 20L, analysisDate)); assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::branchCount, TelemetryData.ProjectStatistics::pullRequestCount, TelemetryData.ProjectStatistics::qualityGate, - TelemetryData.ProjectStatistics::scm, TelemetryData.ProjectStatistics::ci, TelemetryData.ProjectStatistics::devopsPlatform) + .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount, TelemetryData.ProjectStatistics::getQualityGate, + TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi, TelemetryData.ProjectStatistics::getDevopsPlatform, + TelemetryData.ProjectStatistics::getBugs, TelemetryData.ProjectStatistics::getVulnerabilities, TelemetryData.ProjectStatistics::getSecurityHotspots, + TelemetryData.ProjectStatistics::getDevelopmentCost, TelemetryData.ProjectStatistics::getTechnicalDebt) .containsExactlyInAnyOrder( - tuple(1L, 0L, qualityGate1.getUuid(), "scm-1", "ci-1", "azure_devops_cloud"), - tuple(1L, 0L, builtInDefaultQualityGate.getUuid(), "scm-2", "ci-2", "github_cloud")); + tuple(1L, 0L, qualityGate1.getUuid(), "scm-1", "ci-1", "azure_devops_cloud", Optional.of(1L), Optional.of(1L), Optional.of(1L), Optional.of(50.0), Optional.of(5.0)), + tuple(1L, 0L, builtInDefaultQualityGate.getUuid(), "scm-2", "ci-2", "github_cloud", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())); assertThat(data.getQualityGates()) .extracting(TelemetryData.QualityGate::uuid, TelemetryData.QualityGate::isCaycCompliant) .containsExactlyInAnyOrder( @@ -269,8 +292,8 @@ public class TelemetryDataLoaderImplTest { tuple(project.uuid(), "js", 50L), tuple(project.uuid(), "kotlin", 30L)); assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::branchCount, TelemetryData.ProjectStatistics::pullRequestCount, - TelemetryData.ProjectStatistics::scm, TelemetryData.ProjectStatistics::ci) + .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount, + TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi) .containsExactlyInAnyOrder( tuple(2L, 0L, "undetected", "undetected")); } @@ -408,7 +431,7 @@ public class TelemetryDataLoaderImplTest { db.components().insertPublicProject(); TelemetryData data = communityUnderTest.load(); assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::devopsPlatform, TelemetryData.ProjectStatistics::scm, TelemetryData.ProjectStatistics::ci) + .extracting(TelemetryData.ProjectStatistics::getDevopsPlatform, TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi) .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected")); } @@ -430,7 +453,7 @@ public class TelemetryDataLoaderImplTest { db.qualityGates().setDefaultQualityGate(qualityGate); TelemetryData data = communityUnderTest.load(); assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::qualityGate) + .extracting(TelemetryData.ProjectStatistics::getQualityGate) .containsOnly(qualityGate.getUuid()); } |