From c0511e62e7e50def346785d1b6de6cd74059cbb4 Mon Sep 17 00:00:00 2001 From: alain <108417558+alain-kermis-sonarsource@users.noreply.github.com> Date: Thu, 5 Jan 2023 08:51:49 +0100 Subject: [PATCH] SONAR-17768 Move unanalyzed language code telemetry back to global level --- ...anchAnalyzedLanguageCountByProjectDto.java | 17 ---- .../org/sonar/db/component/BranchMapper.xml | 12 +-- .../org/sonar/db/component/BranchDaoTest.java | 9 +- .../sonar/server/telemetry/TelemetryData.java | 37 +++++--- .../telemetry/TelemetryDataJsonWriter.java | 5 +- .../TelemetryDataJsonWriterTest.java | 90 +++++++++++-------- .../telemetry/TelemetryDataLoaderImpl.java | 25 +++--- .../TelemetryDataLoaderImplTest.java | 20 +++-- 8 files changed, 115 insertions(+), 100 deletions(-) diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/PrBranchAnalyzedLanguageCountByProjectDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/PrBranchAnalyzedLanguageCountByProjectDto.java index 63a26d9c6da..532d7742992 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/PrBranchAnalyzedLanguageCountByProjectDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/PrBranchAnalyzedLanguageCountByProjectDto.java @@ -24,8 +24,6 @@ public class PrBranchAnalyzedLanguageCountByProjectDto { private String projectUuid = null; private Long pullRequest = null; private Long branch = null; - private Long unanalyzedCCount = null; - private Long unanalyzedCppCount = null; public String getProjectUuid() { return projectUuid; @@ -51,19 +49,4 @@ public class PrBranchAnalyzedLanguageCountByProjectDto { this.branch = branch; } - public Long getUnanalyzedCCount() { - return unanalyzedCCount; - } - - public void setUnanalyzedCCount(Long unanalyzedCCount) { - this.unanalyzedCCount = unanalyzedCCount; - } - - public Long getUnanalyzedCppCount() { - return unanalyzedCppCount; - } - - public void setUnanalyzedCppCount(Long unanalyzedCppCount) { - this.unanalyzedCppCount = unanalyzedCppCount; - } } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml index fc6074dd865..4f981d9aafe 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml @@ -107,16 +107,8 @@ diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java index 35bdcfc6f15..0740e9c2552 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java @@ -571,12 +571,11 @@ public class BranchDaoTest { db.measures().insertLiveMeasure(project3, unanalyzedC); assertThat(underTest.countPrBranchAnalyzedLanguageByProjectUuid(db.getSession())) - .extracting(PrBranchAnalyzedLanguageCountByProjectDto::getProjectUuid, PrBranchAnalyzedLanguageCountByProjectDto::getBranch, PrBranchAnalyzedLanguageCountByProjectDto::getPullRequest, - PrBranchAnalyzedLanguageCountByProjectDto::getUnanalyzedCCount, PrBranchAnalyzedLanguageCountByProjectDto::getUnanalyzedCppCount) + .extracting(PrBranchAnalyzedLanguageCountByProjectDto::getProjectUuid, PrBranchAnalyzedLanguageCountByProjectDto::getBranch, PrBranchAnalyzedLanguageCountByProjectDto::getPullRequest) .containsExactlyInAnyOrder( - tuple(project1.uuid(), 3L, 3L, 1L, 1L), - tuple(project2.uuid(), 1L, 1L, 0L, 1L), - tuple(project3.uuid(), 2L, 0L, 1L, 0L) + tuple(project1.uuid(), 3L, 3L), + tuple(project2.uuid(), 1L, 1L), + tuple(project3.uuid(), 2L, 0L) ); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java index 2cd82e264e7..d1976434651 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java @@ -47,6 +47,8 @@ public class TelemetryData { private final List users; private final List projects; private final List projectStatistics; + private final Boolean hasUnanalyzedC; + private final Boolean hasUnanalyzedCpp; private final Set customSecurityConfigs; private TelemetryData(Builder builder) { @@ -63,6 +65,8 @@ public class TelemetryData { users = builder.users; projects = builder.projects; projectStatistics = builder.projectStatistics; + hasUnanalyzedC = builder.hasUnanalyzedC; + hasUnanalyzedCpp = builder.hasUnanalyzedCpp; customSecurityConfigs = requireNonNullElse(builder.customSecurityConfigs, Set.of()); } @@ -106,6 +110,14 @@ public class TelemetryData { return isScimEnabled; } + public Optional hasUnanalyzedC() { + return Optional.ofNullable(hasUnanalyzedC); + } + + public Optional hasUnanalyzedCpp() { + return Optional.ofNullable(hasUnanalyzedCpp); + } + public Set getCustomSecurityConfigs() { return customSecurityConfigs; } @@ -137,6 +149,8 @@ public class TelemetryData { private String installationVersion; private boolean inDocker = false; private boolean isScimEnabled; + private Boolean hasUnanalyzedC; + private Boolean hasUnanalyzedCpp; private Set customSecurityConfigs; private List users; private List projects; @@ -191,6 +205,16 @@ public class TelemetryData { return this; } + Builder setHasUnanalyzedC(@Nullable Boolean hasUnanalyzedC) { + this.hasUnanalyzedC = hasUnanalyzedC; + return this; + } + + Builder setHasUnanalyzedCpp(@Nullable Boolean hasUnanalyzedCpp) { + this.hasUnanalyzedCpp = hasUnanalyzedCpp; + return this; + } + Builder setCustomSecurityConfigs(Set customSecurityConfigs) { this.customSecurityConfigs = customSecurityConfigs; return this; @@ -279,19 +303,15 @@ public class TelemetryData { private final String projectUuid; private final Long branchCount; private final Long pullRequestCount; - private final Boolean hasUnanalyzedC; - private final Boolean hasUnanalyzedCpp; private final String scm; private final String ci; private final String devopsPlatform; - ProjectStatistics(String projectUuid, Long branchCount, Long pullRequestCount, @Nullable Boolean hasUnanalyzedC, @Nullable Boolean hasUnanalyzedCpp, + ProjectStatistics(String projectUuid, Long branchCount, Long pullRequestCount, @Nullable String scm, @Nullable String ci, @Nullable String devopsPlatform) { this.projectUuid = projectUuid; this.branchCount = branchCount; this.pullRequestCount = pullRequestCount; - this.hasUnanalyzedC = hasUnanalyzedC; - this.hasUnanalyzedCpp = hasUnanalyzedCpp; this.scm = scm; this.ci = ci; this.devopsPlatform = devopsPlatform; @@ -324,12 +344,5 @@ public class TelemetryData { return devopsPlatform; } - public Optional hasUnanalyzedC() { - return Optional.ofNullable(hasUnanalyzedC); - } - - public Optional hasUnanalyzedCpp() { - return Optional.ofNullable(hasUnanalyzedCpp); - } } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java index a6d12851f9f..fc399f1c23c 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java @@ -77,6 +77,9 @@ public class TelemetryDataJsonWriter { json.endArray(); } + statistics.hasUnanalyzedC().ifPresent(hasUnanalyzedC -> json.prop("hasUnanalyzedC", hasUnanalyzedC)); + statistics.hasUnanalyzedCpp().ifPresent(hasUnanalyzedCpp -> json.prop("hasUnanalyzedCpp", hasUnanalyzedCpp)); + if (statistics.getInstallationDate() != null) { json.prop("installationDate", toUtc(statistics.getInstallationDate())); } @@ -149,8 +152,6 @@ public class TelemetryDataJsonWriter { json.prop("scm", project.getScm()); json.prop("ci", project.getCi()); json.prop("devopsPlatform", project.getDevopsPlatform()); - project.hasUnanalyzedC().ifPresent(hasUnanalyzedC -> json.prop("hasUnanalyzedC", hasUnanalyzedC)); - project.hasUnanalyzedCpp().ifPresent(hasUnanalyzedCpp -> json.prop("hasUnanalyzedCpp", hasUnanalyzedCpp)); json.endObject(); }); json.endArray(); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java index 56604cb8937..1748571360b 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java @@ -224,6 +224,23 @@ public class TelemetryDataJsonWriterTest { assertJson(json).isSimilarTo("{" + format(" \"%s\":", SCIM_PROPERTY) + isScimEnabled + "}"); } + @Test + public void writes_has_unanalyzed_languages() { + TelemetryData data = telemetryBuilder() + .setHasUnanalyzedC(true) + .setHasUnanalyzedCpp(false) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo(""" + { + "hasUnanalyzedC": true, + "hasUnanalyzedCpp": false, + } + """); + } + @Test public void writes_security_custom_config() { TelemetryData data = telemetryBuilder() @@ -319,51 +336,48 @@ public class TelemetryDataJsonWriterTest { @Test public void writes_all_projects_stats_with_analyzed_languages() { TelemetryData data = telemetryBuilder() - .setProjectStatistics(attachProjectStats(true)) + .setProjectStatistics(attachProjectStats()) .build(); String json = writeTelemetryData(data); - assertJson(json).isSimilarTo("{" + - " \"projects-general-stats\": [" + - " {" + - " \"projectUuid\": \"uuid-0\"," + - " \"branchCount\": 2," + - " \"pullRequestCount\": 2," + - " \"scm\": \"scm-0\"," + - " \"ci\": \"ci-0\"," + - " \"devopsPlatform\": \"devops-0\"," + - " \"hasUnanalyzedC\": true," + - " \"hasUnanalyzedCpp\": false" + - " }," + - " {" + - " \"projectUuid\": \"uuid-1\"," + - " \"branchCount\": 4," + - " \"pullRequestCount\": 4," + - " \"scm\": \"scm-1\"," + - " \"ci\": \"ci-1\"," + - " \"devopsPlatform\": \"devops-1\"," + - " \"hasUnanalyzedC\": false," + - " \"hasUnanalyzedCpp\": true" + - " }," + - " {" + - " \"projectUuid\": \"uuid-2\"," + - " \"branchCount\": 6," + - " \"pullRequestCount\": 6," + - " \"scm\": \"scm-2\"," + - " \"ci\": \"ci-2\"," + - " \"devopsPlatform\": \"devops-2\"," + - " \"hasUnanalyzedC\": true," + - " \"hasUnanalyzedCpp\": false" + - " }" + - " ]" + - "}"); + assertJson(json).isSimilarTo(""" + { + "projects-general-stats": [ + { + "projectUuid": "uuid-0", + "branchCount": 2, + "pullRequestCount": 2, + "scm": "scm-0", + "ci": "ci-0", + "devopsPlatform": "devops-0" + }, + { + "projectUuid": "uuid-1", + "branchCount": 4, + "pullRequestCount": 4, + "scm": "scm-1", + "ci": "ci-1", + "devopsPlatform": "devops-1" + }, + { + "projectUuid": "uuid-2", + "branchCount": 6, + "pullRequestCount": 6, + "scm": "scm-2", + "ci": "ci-2", + "devopsPlatform": "devops-2" + } + ] + } + """ + ); } @Test public void writes_all_projects_stats_with_unanalyzed_languages() { TelemetryData data = telemetryBuilder() - .setProjectStatistics(attachProjectStats(false)) + .setProjectStatistics(attachProjectStats()) .build(); String json = writeTelemetryData(data); @@ -391,8 +405,8 @@ public class TelemetryDataJsonWriterTest { return IntStream.range(0, 3).mapToObj(i -> new TelemetryData.Project("uuid-" + i, 1L, "lang-" + i, (i + 1L) * 2L)).collect(Collectors.toList()); } - private List attachProjectStats(boolean hasUnanalyzedLanguages) { - return IntStream.range(0, 3).mapToObj(i -> new TelemetryData.ProjectStatistics("uuid-" + i, (i + 1L) * 2L, (i + 1L) * 2L, hasUnanalyzedLanguages ? i % 2 == 0 : null, hasUnanalyzedLanguages ? i % 2 != 0 : null, "scm-" + i, "ci-" + i, "devops-" + i)) + private List attachProjectStats() { + return IntStream.range(0, 3).mapToObj(i -> new TelemetryData.ProjectStatistics("uuid-" + i, (i + 1L) * 2L, (i + 1L) * 2L, "scm-" + i, "ci-" + i, "devops-" + i)) .collect(Collectors.toList()); } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java index cdf355177ba..3fee8bf14e0 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java @@ -57,6 +57,8 @@ import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETEC import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY; import static org.sonar.core.platform.EditionProvider.Edition.DATACENTER; import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE; +import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY; +import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY; import static org.sonar.server.telemetry.TelemetryDaemon.I_PROP_MESSAGE_SEQUENCE; @ServerSide @@ -115,6 +117,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { try (DbSession dbSession = dbClient.openSession(false)) { data.setDatabase(loadDatabaseMetadata(dbSession)); + resolveUnanalyzedLanguageCode(data, dbSession); resolveProjectStatistics(data, dbSession); resolveProjects(data, dbSession); resolveUsers(data, dbSession); @@ -133,6 +136,17 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { .build(); } + private void resolveUnanalyzedLanguageCode(TelemetryData.Builder data, DbSession dbSession) { + long numberOfUnanalyzedCMeasures = dbClient.liveMeasureDao().countProjectsHavingMeasure(dbSession, UNANALYZED_C_KEY); + long numberOfUnanalyzedCppMeasures = dbClient.liveMeasureDao().countProjectsHavingMeasure(dbSession, UNANALYZED_CPP_KEY); + editionProvider.get() + .filter(edition -> edition.equals(COMMUNITY)) + .ifPresent(edition -> { + data.setHasUnanalyzedC(numberOfUnanalyzedCMeasures > 0); + data.setHasUnanalyzedCpp(numberOfUnanalyzedCppMeasures > 0); + }); + } + private Long retrieveCurrentMessageSequenceNumber() { return internalProperties.read(I_PROP_MESSAGE_SEQUENCE).map(Long::parseLong).orElse(0L); } @@ -145,21 +159,12 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { Map prAndBranchCountByProjects = dbClient.branchDao().countPrBranchAnalyzedLanguageByProjectUuid(dbSession) .stream().collect(Collectors.toMap(PrBranchAnalyzedLanguageCountByProjectDto::getProjectUuid, Function.identity())); - boolean isCommunityEdition = editionProvider.get().filter(edition -> edition.equals(COMMUNITY)).isPresent(); List projectStatistics = new ArrayList<>(); for (String projectUuid : projectUuids) { Optional counts = ofNullable(prAndBranchCountByProjects.get(projectUuid)); Long branchCount = counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getBranch).orElse(0L); Long pullRequestCount = counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getPullRequest).orElse(0L); - - Boolean hasUnanalyzedCMeasures = null; - Boolean hasUnanalyzedCppMeasures = null; - if (isCommunityEdition) { - hasUnanalyzedCMeasures = counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getUnanalyzedCCount).orElse(0L) > 0; - hasUnanalyzedCppMeasures = counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getUnanalyzedCppCount).orElse(0L) > 0; - } - String scm = Optional.ofNullable(scmByProject.get(projectUuid)).orElse(UNDETECTED); String ci = Optional.ofNullable(ciByProject.get(projectUuid)).orElse(UNDETECTED); String devopsPlatform = null; @@ -170,7 +175,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { devopsPlatform = Optional.ofNullable(devopsPlatform).orElse(UNDETECTED); projectStatistics.add( - new TelemetryData.ProjectStatistics(projectUuid, branchCount, pullRequestCount, hasUnanalyzedCMeasures, hasUnanalyzedCppMeasures, scm, ci, devopsPlatform)); + new TelemetryData.ProjectStatistics(projectUuid, branchCount, pullRequestCount, scm, ci, devopsPlatform)); } data.setProjectStatistics(projectStatistics); } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java index 8bab7ca167d..2622b60178e 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java @@ -318,9 +318,8 @@ public class TelemetryDataLoaderImplTest { TelemetryData data = communityUnderTest.load(); - assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::hasUnanalyzedC, TelemetryData.ProjectStatistics::hasUnanalyzedCpp) - .containsExactlyInAnyOrder(tuple(Optional.of(true), Optional.of(true)), tuple(Optional.of(true), Optional.of(false))); + assertThat(data.hasUnanalyzedC().get()).isTrue(); + assertThat(data.hasUnanalyzedCpp().get()).isTrue(); } @Test @@ -335,9 +334,18 @@ public class TelemetryDataLoaderImplTest { TelemetryData data = communityUnderTest.load(); - assertThat(data.getProjectStatistics()) - .extracting(TelemetryData.ProjectStatistics::hasUnanalyzedC, TelemetryData.ProjectStatistics::hasUnanalyzedCpp) - .containsExactlyInAnyOrder(tuple(Optional.empty(), Optional.empty()), tuple(Optional.empty(), Optional.empty())); + assertThat(data.hasUnanalyzedC()).isEmpty(); + assertThat(data.hasUnanalyzedCpp()).isEmpty(); + } + + @Test + public void unanalyzed_languages_flags_are_set_to_false_when_no_unanalyzed_languages_and_edition_is_community() { + when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY)); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.hasUnanalyzedC().get()).isFalse(); + assertThat(data.hasUnanalyzedCpp().get()).isFalse(); } @Test -- 2.39.5