diff options
author | Alain Kermis <alain.kermis@sonarsource.com> | 2024-07-08 17:45:13 +0200 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2024-07-24 20:02:47 +0000 |
commit | ed02521046020a37140eb84ec536c5cdf31e6d4b (patch) | |
tree | f772ac842043dc91aa412adc52e7fbd99b07d13f /server/sonar-telemetry/src | |
parent | 2ab4e9ead92e6f6a64152424303b0d7f0feffe8a (diff) | |
download | sonarqube-ed02521046020a37140eb84ec536c5cdf31e6d4b.tar.gz sonarqube-ed02521046020a37140eb84ec536c5cdf31e6d4b.zip |
SONAR-22479 Create new telemetry module
Diffstat (limited to 'server/sonar-telemetry/src')
18 files changed, 4480 insertions, 0 deletions
diff --git a/server/sonar-telemetry/src/it/java/org/sonar/telemetry/deprecated/QualityProfileDataProviderIT.java b/server/sonar-telemetry/src/it/java/org/sonar/telemetry/deprecated/QualityProfileDataProviderIT.java new file mode 100644 index 00000000000..3da1cf0ca83 --- /dev/null +++ b/server/sonar-telemetry/src/it/java/org/sonar/telemetry/deprecated/QualityProfileDataProviderIT.java @@ -0,0 +1,169 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import javax.annotation.Nullable; +import org.assertj.core.api.Assertions; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.db.DbClient; +import org.sonar.db.DbTester; +import org.sonar.db.component.ProjectData; +import org.sonar.db.qualityprofile.ActiveRuleDto; +import org.sonar.db.qualityprofile.ActiveRuleParamDto; +import org.sonar.db.qualityprofile.QProfileDto; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleParamDto; +import org.sonar.server.qualityprofile.QProfileComparison; + +import static org.assertj.core.groups.Tuple.tuple; +import static org.sonar.db.qualityprofile.ActiveRuleDto.OVERRIDES; + +public class QualityProfileDataProviderIT { + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + private DbClient dbClient = dbTester.getDbClient(); + + QualityProfileDataProvider underTest = new QualityProfileDataProvider(dbClient, new QProfileComparison(dbClient)); + + @Test + public void retrieveQualityProfilesData_whenDefaultRootProfile_shouldReturnRelevantInformation() { + QProfileDto qProfile1 = createQualityProfile(false, null); + dbTester.qualityProfiles().setAsDefault(qProfile1); + Assertions.assertThat(underTest.retrieveQualityProfilesData()) + .extracting(p -> p.uuid(), p -> p.isDefault(), p -> p.isBuiltIn(), p -> p.builtInParent(), + p -> p.rulesActivatedCount(), p -> p.rulesDeactivatedCount(), p -> p.rulesOverriddenCount()) + .containsExactlyInAnyOrder(tuple(qProfile1.getKee(), true, false, false, null, null, null)); + } + + @Test + public void retrieveQualityProfilesData_whenDefaultChildProfile_shouldReturnRelevantInformation() { + QProfileDto rootProfile = createQualityProfile(false, null); + + QProfileDto childProfile = createQualityProfile(false, rootProfile.getKee()); + + dbTester.qualityProfiles().setAsDefault(childProfile); + Assertions.assertThat(underTest.retrieveQualityProfilesData()) + .extracting(p -> p.uuid(), p -> p.isDefault(), p -> p.isBuiltIn(), p -> p.builtInParent(), + p -> p.rulesActivatedCount(), p -> p.rulesDeactivatedCount(), p -> p.rulesOverriddenCount()) + .containsExactlyInAnyOrder( + tuple(rootProfile.getKee(), false, false, false, null, null, null), + tuple(childProfile.getKee(), true, false, false, null, null, null)); + } + + @Test + public void retrieveQualityProfilesData_whenProfileAssignedToProject_shouldReturnProfile() { + ProjectData projectData = dbTester.components().insertPublicProject(); + + QProfileDto associatedProfile = createQualityProfile(false, null); + + QProfileDto unassociatedProfile = createQualityProfile(false, null); + + dbTester.qualityProfiles().associateWithProject(projectData.getProjectDto(), associatedProfile); + + Assertions.assertThat(underTest.retrieveQualityProfilesData()) + .extracting(p -> p.uuid(), p -> p.isDefault()) + .containsExactlyInAnyOrder( + tuple(associatedProfile.getKee(), false), + tuple(unassociatedProfile.getKee(), false) + ); + } + + @Test + public void retrieveQualityProfilesData_whenBuiltInParent_shouldReturnBuiltInParent() { + + QProfileDto rootBuiltinProfile = createQualityProfile(true, null); + + QProfileDto childProfile = createQualityProfile(false, rootBuiltinProfile.getKee()); + + QProfileDto grandChildProfile = createQualityProfile(false, childProfile.getKee()); + + dbTester.qualityProfiles().setAsDefault(rootBuiltinProfile, childProfile, grandChildProfile); + + Assertions.assertThat(underTest.retrieveQualityProfilesData()) + .extracting(p -> p.uuid(), p -> p.isBuiltIn(), p -> p.builtInParent()) + .containsExactlyInAnyOrder(tuple(rootBuiltinProfile.getKee(), true, null), + tuple(childProfile.getKee(), false, true), + tuple(grandChildProfile.getKee(), false, true) + ); + } + + @Test + public void retrieveQualityProfilesData_whenBuiltInParent_shouldReturnActiveAndUnactiveRules() { + + QProfileDto rootBuiltinProfile = createQualityProfile(true, null); + + QProfileDto childProfile = createQualityProfile(false, rootBuiltinProfile.getKee()); + RuleDto activatedRule = dbTester.rules().insert(); + RuleDto deactivatedRule = dbTester.rules().insert(); + + dbTester.qualityProfiles().activateRule(rootBuiltinProfile, deactivatedRule); + dbTester.qualityProfiles().activateRule(childProfile, activatedRule); + dbTester.qualityProfiles().setAsDefault(childProfile); + + Assertions.assertThat(underTest.retrieveQualityProfilesData()) + .extracting(p -> p.uuid(), p -> p.rulesActivatedCount(), p -> p.rulesDeactivatedCount(), p -> p.rulesOverriddenCount()) + .containsExactlyInAnyOrder( + tuple(rootBuiltinProfile.getKee(), null, null, null), + tuple(childProfile.getKee(), 1, 1, 0) + ); + } + + @Test + public void retrieveQualityProfilesData_whenBuiltInParent_shouldReturnOverriddenRules() { + + QProfileDto rootBuiltinProfile = createQualityProfile(true, null); + + QProfileDto childProfile = createQualityProfile(false, rootBuiltinProfile.getKee()); + RuleDto rule = dbTester.rules().insert(); + RuleParamDto initialRuleParam = dbTester.rules().insertRuleParam(rule, p -> p.setName("key").setDefaultValue("initial")); + + + ActiveRuleDto activeRuleDto = dbTester.qualityProfiles().activateRule(rootBuiltinProfile, rule); + dbTester.getDbClient().activeRuleDao().insertParam(dbTester.getSession(), activeRuleDto, newParam(activeRuleDto, initialRuleParam, "key", "value")); + + ActiveRuleDto childActivateRule = dbTester.qualityProfiles().activateRule(childProfile, rule, ar -> { + ar.setInheritance(OVERRIDES); + }); + dbTester.getDbClient().activeRuleDao().insertParam(dbTester.getSession(), activeRuleDto, newParam(childActivateRule, initialRuleParam, "key", "override")); + + dbTester.qualityProfiles().setAsDefault(childProfile); + + Assertions.assertThat(underTest.retrieveQualityProfilesData()) + .extracting(p -> p.uuid(), p -> p.rulesActivatedCount(), p -> p.rulesDeactivatedCount(), p -> p.rulesOverriddenCount()) + .containsExactlyInAnyOrder( + tuple(rootBuiltinProfile.getKee(), null, null, null), + tuple(childProfile.getKee(), 0, 0, 1)); + } + + private static ActiveRuleParamDto newParam(ActiveRuleDto activeRuleDto, RuleParamDto initial, String key, String value) { + return new ActiveRuleParamDto().setActiveRuleUuid(activeRuleDto.getRuleUuid()).setRulesParameterUuid(initial.getUuid()).setKey(key).setValue(value); + } + + private QProfileDto createQualityProfile(boolean isBuiltIn, @Nullable String parentKee) { + return dbTester.qualityProfiles().insert(p -> { + p.setIsBuiltIn(isBuiltIn); + p.setParentKee(parentKee); + }); + } +} diff --git a/server/sonar-telemetry/src/it/java/org/sonar/telemetry/deprecated/TelemetryDataLoaderImplIT.java b/server/sonar-telemetry/src/it/java/org/sonar/telemetry/deprecated/TelemetryDataLoaderImplIT.java new file mode 100644 index 00000000000..4f47a549ee9 --- /dev/null +++ b/server/sonar-telemetry/src/it/java/org/sonar/telemetry/deprecated/TelemetryDataLoaderImplIT.java @@ -0,0 +1,742 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.IntStream; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.sonar.api.config.Configuration; +import org.sonar.api.impl.utils.TestSystem2; +import org.sonar.core.platform.PlatformEditionProvider; +import org.sonar.core.platform.PluginInfo; +import org.sonar.core.platform.PluginRepository; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.alm.setting.AlmSettingDto; +import org.sonar.db.component.AnalysisPropertyDto; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ProjectData; +import org.sonar.db.component.SnapshotDto; +import org.sonar.db.metric.MetricDto; +import org.sonar.db.newcodeperiod.NewCodePeriodType; +import org.sonar.db.project.CreationMethod; +import org.sonar.db.property.PropertyDto; +import org.sonar.db.qualitygate.QualityGateConditionDto; +import org.sonar.db.qualitygate.QualityGateDto; +import org.sonar.db.qualityprofile.QProfileDto; +import org.sonar.db.user.UserDbTester; +import org.sonar.db.user.UserDto; +import org.sonar.db.user.UserTelemetryDto; +import org.sonar.server.management.ManagedInstanceService; +import org.sonar.server.platform.ContainerSupport; +import org.sonar.server.property.InternalProperties; +import org.sonar.server.property.MapInternalProperties; +import org.sonar.server.qualitygate.QualityGateCaycChecker; +import org.sonar.server.qualitygate.QualityGateFinder; +import org.sonar.server.qualityprofile.QProfileComparison; +import org.sonar.telemetry.deprecated.TelemetryData.Branch; +import org.sonar.telemetry.deprecated.TelemetryData.CloudUsage; +import org.sonar.telemetry.deprecated.TelemetryData.NewCodeDefinition; +import org.sonar.telemetry.deprecated.TelemetryData.ProjectStatistics; +import org.sonar.updatecenter.common.Version; + +import static java.util.Arrays.asList; +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static org.assertj.core.groups.Tuple.tuple; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.sonar.api.measures.CoreMetrics.ALERT_STATUS_KEY; +import static org.sonar.api.measures.CoreMetrics.BUGS_KEY; +import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY; +import static org.sonar.api.measures.CoreMetrics.DEVELOPMENT_COST_KEY; +import static org.sonar.api.measures.CoreMetrics.LINES_KEY; +import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; +import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_KEY; +import static org.sonar.api.measures.CoreMetrics.TECHNICAL_DEBT_KEY; +import static org.sonar.api.measures.CoreMetrics.VULNERABILITIES_KEY; +import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI; +import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM; +import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY; +import static org.sonar.core.platform.EditionProvider.Edition.DEVELOPER; +import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE; +import static org.sonar.db.component.BranchType.BRANCH; +import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY; +import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY; +import static org.sonar.server.qualitygate.QualityGateCaycStatus.NON_COMPLIANT; +import static org.sonar.telemetry.deprecated.TelemetryDataLoaderImpl.EXTERNAL_SECURITY_REPORT_EXPORTED_AT; + +@RunWith(DataProviderRunner.class) +public class TelemetryDataLoaderImplIT { + private final static Long NOW = 100_000_000L; + public static final String SERVER_ID = "AU-TpxcB-iU5OvuD2FL7"; + private final TestSystem2 system2 = new TestSystem2().setNow(NOW); + + @Rule + public DbTester db = DbTester.create(system2); + + private final FakeServer server = new FakeServer(); + private final PluginRepository pluginRepository = mock(PluginRepository.class); + private final Configuration configuration = mock(Configuration.class); + private final PlatformEditionProvider editionProvider = mock(PlatformEditionProvider.class); + private final ContainerSupport containerSupport = mock(ContainerSupport.class); + private final QualityGateCaycChecker qualityGateCaycChecker = mock(QualityGateCaycChecker.class); + private final QualityGateFinder qualityGateFinder = new QualityGateFinder(db.getDbClient()); + + private final QualityProfileDataProvider qualityProfileDataProvider = new QualityProfileDataProvider(db.getDbClient(), new QProfileComparison(db.getDbClient())); + private final InternalProperties internalProperties = spy(new MapInternalProperties()); + private final ManagedInstanceService managedInstanceService = mock(ManagedInstanceService.class); + private final CloudUsageDataProvider cloudUsageDataProvider = mock(CloudUsageDataProvider.class); + + private final TelemetryDataLoader communityUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider, + internalProperties, configuration, containerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService, cloudUsageDataProvider, qualityProfileDataProvider); + private final TelemetryDataLoader commercialUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider, + internalProperties, configuration, containerSupport, qualityGateCaycChecker, qualityGateFinder, managedInstanceService, cloudUsageDataProvider, qualityProfileDataProvider); + + private QualityGateDto builtInDefaultQualityGate; + private MetricDto bugsDto; + private MetricDto vulnerabilitiesDto; + private MetricDto securityHotspotsDto; + private MetricDto technicalDebtDto; + private MetricDto developmentCostDto; + + @Before + public void setUpBuiltInQualityGate() { + String builtInQgName = "Sonar way"; + builtInDefaultQualityGate = db.qualityGates().insertQualityGate(qg -> qg.setName(builtInQgName).setBuiltIn(true)); + when(qualityGateCaycChecker.checkCaycCompliant(any(), any())).thenReturn(NON_COMPLIANT); + db.qualityGates().setDefaultQualityGate(builtInDefaultQualityGate); + + bugsDto = db.measures().insertMetric(m -> m.setKey(BUGS_KEY)); + vulnerabilitiesDto = db.measures().insertMetric(m -> m.setKey(VULNERABILITIES_KEY)); + securityHotspotsDto = db.measures().insertMetric(m -> m.setKey(SECURITY_HOTSPOTS_KEY)); + technicalDebtDto = db.measures().insertMetric(m -> m.setKey(TECHNICAL_DEBT_KEY)); + developmentCostDto = db.measures().insertMetric(m -> m.setKey(DEVELOPMENT_COST_KEY)); + } + + @Test + public void send_telemetry_data() { + String version = "7.5.4"; + Long analysisDate = 1L; + Long lastConnectionDate = 5L; + + server.setId(SERVER_ID); + server.setVersion(version); + List<PluginInfo> plugins = asList(newPlugin("java", "4.12.0.11033"), newPlugin("scmgit", "1.2"), new PluginInfo("other")); + when(pluginRepository.getPluginInfos()).thenReturn(plugins); + when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER)); + + List<UserDto> activeUsers = composeActiveUsers(3); + + // update last connection + activeUsers.forEach(u -> db.users().updateLastConnectionDate(u, 5L)); + + UserDto inactiveUser = db.users().insertUser(u -> u.setActive(false).setExternalIdentityProvider("provider0")); + + MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY)); + MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY)); + MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY)); + MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY)); + + ProjectData projectData1 = db.components().insertPrivateProject(); + ComponentDto mainBranch1 = projectData1.getMainBranchComponent(); + var branch1 = db.components().insertProjectBranch(mainBranch1, branchDto -> branchDto.setKey("reference")); + var branch2 = db.components().insertProjectBranch(mainBranch1, branchDto -> branchDto.setKey("custom")); + db.measures().insertLiveMeasure(mainBranch1, lines, m -> m.setValue(110d)); + db.measures().insertLiveMeasure(mainBranch1, ncloc, m -> m.setValue(110d)); + db.measures().insertLiveMeasure(mainBranch1, coverage, m -> m.setValue(80d)); + db.measures().insertLiveMeasure(mainBranch1, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10")); + db.measures().insertLiveMeasure(mainBranch1, bugsDto, m -> m.setValue(1d)); + db.measures().insertLiveMeasure(mainBranch1, vulnerabilitiesDto, m -> m.setValue(1d).setData((String) null)); + db.measures().insertLiveMeasure(mainBranch1, securityHotspotsDto, m -> m.setValue(1d).setData((String) null)); + db.measures().insertLiveMeasure(mainBranch1, developmentCostDto, m -> m.setData("50").setValue(null)); + db.measures().insertLiveMeasure(mainBranch1, technicalDebtDto, m -> m.setValue(5d).setData((String) null)); + // Measures on other branches + db.measures().insertLiveMeasure(branch1, technicalDebtDto, m -> m.setValue(6d).setData((String) null)); + db.measures().insertLiveMeasure(branch2, technicalDebtDto, m -> m.setValue(7d).setData((String) null)); + + ProjectData projectData2 = db.components().insertPrivateProject(); + ComponentDto mainBranch2 = projectData2.getMainBranchComponent(); + db.measures().insertLiveMeasure(mainBranch2, lines, m -> m.setValue(200d)); + db.measures().insertLiveMeasure(mainBranch2, ncloc, m -> m.setValue(200d)); + db.measures().insertLiveMeasure(mainBranch2, coverage, m -> m.setValue(80d)); + db.measures().insertLiveMeasure(mainBranch2, nclocDistrib, m -> m.setValue(null).setData("java=180;js=20")); + + SnapshotDto project1Analysis = db.components().insertSnapshot(mainBranch1, t -> t.setLast(true).setAnalysisDate(analysisDate)); + SnapshotDto project2Analysis = db.components().insertSnapshot(mainBranch2, t -> t.setLast(true).setAnalysisDate(analysisDate)); + db.measures().insertMeasure(mainBranch1, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10")); + db.measures().insertMeasure(mainBranch2, project2Analysis, nclocDistrib, m -> m.setData("java=180;js=20")); + + insertAnalysisProperty(project1Analysis, "prop-uuid-1", SONAR_ANALYSIS_DETECTEDCI, "ci-1"); + insertAnalysisProperty(project2Analysis, "prop-uuid-2", SONAR_ANALYSIS_DETECTEDCI, "ci-2"); + insertAnalysisProperty(project1Analysis, "prop-uuid-3", SONAR_ANALYSIS_DETECTEDSCM, "scm-1"); + insertAnalysisProperty(project2Analysis, "prop-uuid-4", SONAR_ANALYSIS_DETECTEDSCM, "scm-2"); + + // alm + db.almSettings().insertAzureAlmSetting(); + db.almSettings().insertGitHubAlmSetting(); + AlmSettingDto almSettingDto = db.almSettings().insertAzureAlmSetting(a -> a.setUrl("https://dev.azure.com")); + AlmSettingDto gitHubAlmSetting = db.almSettings().insertGitHubAlmSetting(a -> a.setUrl("https://api.github.com")); + db.almSettings().insertAzureProjectAlmSetting(almSettingDto, projectData1.getProjectDto()); + db.almSettings().insertGitlabProjectAlmSetting(gitHubAlmSetting, projectData2.getProjectDto(), true); + + // quality gates + QualityGateDto qualityGate1 = db.qualityGates().insertQualityGate(qg -> qg.setName("QG1").setBuiltIn(true)); + QualityGateDto qualityGate2 = db.qualityGates().insertQualityGate(qg -> qg.setName("QG2")); + + QualityGateConditionDto condition1 = db.qualityGates().addCondition(qualityGate1, vulnerabilitiesDto, c -> c.setOperator("GT").setErrorThreshold("80")); + QualityGateConditionDto condition2 = db.qualityGates().addCondition(qualityGate2, securityHotspotsDto, c -> c.setOperator("LT").setErrorThreshold("2")); + + // quality profiles + QProfileDto javaQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("java")); + QProfileDto kotlinQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("kotlin")); + QProfileDto jsQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("js")); + db.qualityProfiles().associateWithProject(projectData1.getProjectDto(), javaQP, kotlinQP, jsQP); + db.qualityProfiles().associateWithProject(projectData2.getProjectDto(), javaQP, jsQP); + + QProfileDto qualityProfile1 = db.qualityProfiles().insert(qp -> qp.setIsBuiltIn(true)); + QProfileDto qualityProfile2 = db.qualityProfiles().insert(); + db.qualityProfiles().setAsDefault(qualityProfile1, qualityProfile2); + + // link one project to a non-default QG + db.qualityGates().associateProjectToQualityGate(db.components().getProjectDtoByMainBranch(mainBranch1), qualityGate1); + + db.newCodePeriods().insert(projectData1.projectUuid(), NewCodePeriodType.NUMBER_OF_DAYS, "30"); + db.newCodePeriods().insert(projectData1.projectUuid(), branch2.branchUuid(), NewCodePeriodType.REFERENCE_BRANCH, "reference"); + + var instanceNcdId = NewCodeDefinition.getInstanceDefault().hashCode(); + var projectNcdId = new NewCodeDefinition(NewCodePeriodType.NUMBER_OF_DAYS.name(), "30", "project").hashCode(); + var branchNcdId = new NewCodeDefinition(NewCodePeriodType.REFERENCE_BRANCH.name(), branch1.uuid(), "branch").hashCode(); + + TelemetryData data = communityUnderTest.load(); + assertThat(data.getServerId()).isEqualTo(SERVER_ID); + assertThat(data.getVersion()).isEqualTo(version); + assertThat(data.getEdition()).contains(DEVELOPER); + assertThat(data.getDefaultQualityGate()).isEqualTo(builtInDefaultQualityGate.getUuid()); + assertThat(data.getSonarWayQualityGate()).isEqualTo(builtInDefaultQualityGate.getUuid()); + assertThat(data.getNcdId()).isEqualTo(NewCodeDefinition.getInstanceDefault().hashCode()); + assertThat(data.getMessageSequenceNumber()).isOne(); + assertDatabaseMetadata(data.getDatabase()); + assertThat(data.getPlugins()).containsOnly( + entry("java", "4.12.0.11033"), entry("scmgit", "1.2"), entry("other", "undefined")); + assertThat(data.isInContainer()).isFalse(); + + assertThat(data.getUserTelemetries()) + .extracting(UserTelemetryDto::getUuid, UserTelemetryDto::getLastConnectionDate, UserTelemetryDto::getLastSonarlintConnectionDate, UserTelemetryDto::isActive) + .containsExactlyInAnyOrder( + tuple(activeUsers.get(0).getUuid(), lastConnectionDate, activeUsers.get(0).getLastSonarlintConnectionDate(), true), + tuple(activeUsers.get(1).getUuid(), lastConnectionDate, activeUsers.get(1).getLastSonarlintConnectionDate(), true), + tuple(activeUsers.get(2).getUuid(), lastConnectionDate, activeUsers.get(2).getLastSonarlintConnectionDate(), true), + tuple(inactiveUser.getUuid(), null, inactiveUser.getLastSonarlintConnectionDate(), false)); + assertThat(data.getProjects()) + .extracting(TelemetryData.Project::projectUuid, TelemetryData.Project::language, TelemetryData.Project::loc, TelemetryData.Project::lastAnalysis) + .containsExactlyInAnyOrder( + tuple(projectData1.projectUuid(), "java", 70L, analysisDate), + tuple(projectData1.projectUuid(), "js", 30L, analysisDate), + tuple(projectData1.projectUuid(), "kotlin", 10L, analysisDate), + tuple(projectData2.projectUuid(), "java", 180L, analysisDate), + tuple(projectData2.projectUuid(), "js", 20L, analysisDate)); + assertThat(data.getProjectStatistics()) + .extracting( + ProjectStatistics::getBranchCount, + ProjectStatistics::getPullRequestCount, + ProjectStatistics::getQualityGate, + ProjectStatistics::getScm, + ProjectStatistics::getCi, + ProjectStatistics::getDevopsPlatform, + ProjectStatistics::getBugs, + ProjectStatistics::getVulnerabilities, + ProjectStatistics::getSecurityHotspots, + ProjectStatistics::getDevelopmentCost, + ProjectStatistics::getTechnicalDebt, + ProjectStatistics::getNcdId, + ProjectStatistics::isMonorepo) + .containsExactlyInAnyOrder( + tuple(3L, 0L, qualityGate1.getUuid(), "scm-1", "ci-1", "azure_devops_cloud", Optional.of(1L), Optional.of(1L), Optional.of(1L), Optional.of(50L), Optional.of(5L), + projectNcdId, false), + tuple(1L, 0L, builtInDefaultQualityGate.getUuid(), "scm-2", "ci-2", "github_cloud", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), instanceNcdId, true)); + + assertThat(data.getBranches()) + .extracting(Branch::branchUuid, Branch::ncdId) + .containsExactlyInAnyOrder( + tuple(branch1.uuid(), projectNcdId), + tuple(branch2.uuid(), branchNcdId), + tuple(mainBranch1.uuid(), projectNcdId), + tuple(mainBranch2.uuid(), instanceNcdId)); + + assertThat(data.getNewCodeDefinitions()) + .extracting(NewCodeDefinition::scope, NewCodeDefinition::type, NewCodeDefinition::value) + .containsExactlyInAnyOrder( + tuple("instance", NewCodePeriodType.PREVIOUS_VERSION.name(), ""), + tuple("project", NewCodePeriodType.NUMBER_OF_DAYS.name(), "30"), + tuple("branch", NewCodePeriodType.REFERENCE_BRANCH.name(), branch1.uuid())); + + assertThat(data.getQualityGates()) + .extracting(TelemetryData.QualityGate::uuid, TelemetryData.QualityGate::caycStatus, + qg -> qg.conditions().stream() + .map(condition -> tuple(condition.getMetricKey(), condition.getOperator().getDbValue(), condition.getErrorThreshold(), condition.isOnLeakPeriod())) + .toList()) + .containsExactlyInAnyOrder( + tuple(builtInDefaultQualityGate.getUuid(), "non-compliant", Collections.emptyList()), + tuple(qualityGate1.getUuid(), "non-compliant", List.of(tuple(vulnerabilitiesDto.getKey(), condition1.getOperator(), condition1.getErrorThreshold(), false))), + tuple(qualityGate2.getUuid(), "non-compliant", List.of(tuple(securityHotspotsDto.getKey(), condition2.getOperator(), condition2.getErrorThreshold(), false)))); + + assertThat(data.getQualityProfiles()) + .extracting(TelemetryData.QualityProfile::uuid, TelemetryData.QualityProfile::isBuiltIn) + .containsExactlyInAnyOrder( + tuple(qualityProfile1.getKee(), qualityProfile1.isBuiltIn()), + tuple(qualityProfile2.getKee(), qualityProfile2.isBuiltIn()), + tuple(jsQP.getKee(), jsQP.isBuiltIn()), + tuple(javaQP.getKee(), javaQP.isBuiltIn()), + tuple(kotlinQP.getKee(), kotlinQP.isBuiltIn())); + + } + + @Test + public void send_branch_measures_data() { + Long analysisDate = ZonedDateTime.now(ZoneId.systemDefault()).toInstant().toEpochMilli(); + + MetricDto qg = db.measures().insertMetric(m -> m.setKey(ALERT_STATUS_KEY)); + + ProjectData projectData1 = db.components().insertPrivateProject(); + ComponentDto mainBranch1 = projectData1.getMainBranchComponent(); + + ProjectData projectData2 = db.components().insertPrivateProject(); + ComponentDto mainBranch2 = projectData2.getMainBranchComponent(); + + SnapshotDto project1Analysis1 = db.components().insertSnapshot(mainBranch1, t -> t.setLast(true).setAnalysisDate(analysisDate)); + SnapshotDto project1Analysis2 = db.components().insertSnapshot(mainBranch1, t -> t.setLast(true).setAnalysisDate(analysisDate)); + SnapshotDto project2Analysis = db.components().insertSnapshot(mainBranch2, t -> t.setLast(true).setAnalysisDate(analysisDate)); + db.measures().insertMeasure(mainBranch1, project1Analysis1, qg, pm -> pm.setData("OK")); + db.measures().insertMeasure(mainBranch1, project1Analysis2, qg, pm -> pm.setData("ERROR")); + db.measures().insertMeasure(mainBranch2, project2Analysis, qg, pm -> pm.setData("ERROR")); + + var branch1 = db.components().insertProjectBranch(mainBranch1, branchDto -> branchDto.setKey("reference")); + var branch2 = db.components().insertProjectBranch(mainBranch1, branchDto -> branchDto.setKey("custom")); + + db.newCodePeriods().insert(projectData1.projectUuid(), NewCodePeriodType.NUMBER_OF_DAYS, "30"); + db.newCodePeriods().insert(projectData1.projectUuid(), branch2.branchUuid(), NewCodePeriodType.REFERENCE_BRANCH, "reference"); + + var instanceNcdId = NewCodeDefinition.getInstanceDefault().hashCode(); + var projectNcdId = new NewCodeDefinition(NewCodePeriodType.NUMBER_OF_DAYS.name(), "30", "project").hashCode(); + var branchNcdId = new NewCodeDefinition(NewCodePeriodType.REFERENCE_BRANCH.name(), branch1.uuid(), "branch").hashCode(); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.getBranches()) + .extracting(Branch::branchUuid, Branch::ncdId, Branch::greenQualityGateCount, Branch::analysisCount) + .containsExactlyInAnyOrder( + tuple(branch1.uuid(), projectNcdId, 0, 0), + tuple(branch2.uuid(), branchNcdId, 0, 0), + tuple(mainBranch1.uuid(), projectNcdId, 1, 2), + tuple(mainBranch2.uuid(), instanceNcdId, 0, 1)); + + } + + private List<UserDto> composeActiveUsers(int count) { + UserDbTester userDbTester = db.users(); + Function<Integer, Consumer<UserDto>> userConfigurator = index -> user -> user.setExternalIdentityProvider("provider" + index).setLastSonarlintConnectionDate(index * 2L); + + return IntStream + .rangeClosed(1, count) + .mapToObj(userConfigurator::apply) + .map(userDbTester::insertUser) + .toList(); + } + + private void assertDatabaseMetadata(TelemetryData.Database database) { + try (DbSession dbSession = db.getDbClient().openSession(false)) { + DatabaseMetaData metadata = dbSession.getConnection().getMetaData(); + assertThat(database.name()).isEqualTo("H2"); + assertThat(database.version()).isEqualTo(metadata.getDatabaseProductVersion()); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Test + public void take_largest_branch_snapshot_project_data() { + server.setId(SERVER_ID).setVersion("7.5.4"); + + MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY)); + MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY)); + MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY)); + MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY)); + + ProjectData projectData = db.components().insertPublicProject(); + + QProfileDto javaQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("java")); + QProfileDto kotlinQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("kotlin")); + QProfileDto jsQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("js")); + db.qualityProfiles().associateWithProject(projectData.getProjectDto(), javaQP, kotlinQP, jsQP); + + ComponentDto mainBranch = projectData.getMainBranchComponent(); + db.measures().insertLiveMeasure(mainBranch, lines, m -> m.setValue(110d)); + db.measures().insertLiveMeasure(mainBranch, ncloc, m -> m.setValue(110d)); + db.measures().insertLiveMeasure(mainBranch, coverage, m -> m.setValue(80d)); + db.measures().insertLiveMeasure(mainBranch, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10")); + + ComponentDto branch = db.components().insertProjectBranch(mainBranch, b -> b.setBranchType(BRANCH)); + db.measures().insertLiveMeasure(branch, lines, m -> m.setValue(180d)); + db.measures().insertLiveMeasure(branch, ncloc, m -> m.setValue(180d)); + db.measures().insertLiveMeasure(branch, coverage, m -> m.setValue(80d)); + db.measures().insertLiveMeasure(branch, nclocDistrib, m -> m.setValue(null).setData("java=100;js=50;kotlin=30")); + + SnapshotDto project1Analysis = db.components().insertSnapshot(mainBranch, t -> t.setLast(true)); + SnapshotDto project2Analysis = db.components().insertSnapshot(branch, t -> t.setLast(true)); + db.measures().insertMeasure(mainBranch, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10")); + db.measures().insertMeasure(branch, project2Analysis, nclocDistrib, m -> m.setData("java=100;js=50;kotlin=30")); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.getProjects()).extracting(TelemetryData.Project::projectUuid, TelemetryData.Project::language, TelemetryData.Project::loc) + .containsExactlyInAnyOrder( + tuple(projectData.projectUuid(), "java", 100L), + tuple(projectData.projectUuid(), "js", 50L), + tuple(projectData.projectUuid(), "kotlin", 30L)); + assertThat(data.getProjectStatistics()) + .extracting(ProjectStatistics::getBranchCount, ProjectStatistics::getPullRequestCount, + ProjectStatistics::getScm, ProjectStatistics::getCi) + .containsExactlyInAnyOrder( + tuple(2L, 0L, "undetected", "undetected")); + } + + @Test + public void load_shouldProvideQualityProfileInProjectSection() { + server.setId(SERVER_ID).setVersion("7.5.4"); + MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY)); + MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY)); + + ProjectData projectData = db.components().insertPublicProject(); + + // default quality profile + QProfileDto javaQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("java")); + QProfileDto kotlinQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("kotlin")); + db.qualityProfiles().setAsDefault(javaQP, kotlinQP); + // selected quality profile + QProfileDto jsQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("js")); + db.qualityProfiles().associateWithProject(projectData.getProjectDto(), jsQP); + + ComponentDto mainBranch = projectData.getMainBranchComponent(); + db.measures().insertLiveMeasure(mainBranch, ncloc, m -> m.setValue(110d)); + db.measures().insertLiveMeasure(mainBranch, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10")); + + ComponentDto branch = db.components().insertProjectBranch(mainBranch, b -> b.setBranchType(BRANCH)); + db.measures().insertLiveMeasure(branch, ncloc, m -> m.setValue(180d)); + db.measures().insertLiveMeasure(branch, nclocDistrib, m -> m.setValue(null).setData("java=100;js=50;kotlin=30")); + + SnapshotDto project1Analysis = db.components().insertSnapshot(mainBranch, t -> t.setLast(true)); + SnapshotDto project2Analysis = db.components().insertSnapshot(branch, t -> t.setLast(true)); + db.measures().insertMeasure(mainBranch, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10")); + db.measures().insertMeasure(branch, project2Analysis, nclocDistrib, m -> m.setData("java=100;js=50;kotlin=30")); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.getProjects()).extracting(TelemetryData.Project::projectUuid, TelemetryData.Project::language, TelemetryData.Project::qualityProfile) + .containsExactlyInAnyOrder( + tuple(projectData.projectUuid(), "java", javaQP.getKee()), + tuple(projectData.projectUuid(), "js", jsQP.getKee()), + tuple(projectData.projectUuid(), "kotlin", kotlinQP.getKee())); + } + + @Test + public void load_shouldProvideCreationMethodInProjectStatisticsSection() { + server.setId(SERVER_ID).setVersion("7.5.4"); + + ProjectData projectData1 = db.components().insertPrivateProjectWithCreationMethod(CreationMethod.LOCAL_API); + ProjectData projectData2 = db.components().insertPrivateProjectWithCreationMethod(CreationMethod.LOCAL_BROWSER); + ProjectData projectData3 = db.components().insertPrivateProjectWithCreationMethod(CreationMethod.UNKNOWN); + ProjectData projectData4 = db.components().insertPrivateProjectWithCreationMethod(CreationMethod.SCANNER_API); + ProjectData projectData5 = db.components().insertPrivateProjectWithCreationMethod(CreationMethod.ALM_IMPORT_BROWSER); + ProjectData projectData6 = db.components().insertPrivateProjectWithCreationMethod(CreationMethod.ALM_IMPORT_API); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.getProjectStatistics()).extracting(TelemetryData.ProjectStatistics::getProjectUuid, TelemetryData.ProjectStatistics::getCreationMethod) + .containsExactlyInAnyOrder( + tuple(projectData1.projectUuid(), CreationMethod.LOCAL_API), + tuple(projectData2.projectUuid(), CreationMethod.LOCAL_BROWSER), + tuple(projectData3.projectUuid(), CreationMethod.UNKNOWN), + tuple(projectData4.projectUuid(), CreationMethod.SCANNER_API), + tuple(projectData5.projectUuid(), CreationMethod.ALM_IMPORT_BROWSER), + tuple(projectData6.projectUuid(), CreationMethod.ALM_IMPORT_API)); + } + + @Test + public void test_ncd_on_community_edition() { + server.setId(SERVER_ID).setVersion("7.5.4"); + when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY)); + + ProjectData project = db.components().insertPublicProject(); + + ComponentDto branch = db.components().insertProjectBranch(project.getMainBranchComponent(), b -> b.setBranchType(BRANCH)); + + db.newCodePeriods().insert(project.projectUuid(), branch.branchUuid(), NewCodePeriodType.NUMBER_OF_DAYS, "30"); + + var projectNcdId = new NewCodeDefinition(NewCodePeriodType.NUMBER_OF_DAYS.name(), "30", "project").hashCode(); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.getProjectStatistics()) + .extracting(ProjectStatistics::getBranchCount, ProjectStatistics::getNcdId) + .containsExactlyInAnyOrder(tuple(2L, projectNcdId)); + + assertThat(data.getBranches()) + .extracting(Branch::branchUuid, Branch::ncdId) + .contains(tuple(branch.uuid(), projectNcdId)); + } + + @Test + public void data_contains_weekly_count_sonarlint_users() { + db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 100_000L)); + db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW)); + // these don't count + db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 1_000_000_000L)); + db.users().insertUser(); + + TelemetryData data = communityUnderTest.load(); + assertThat(data.getUserTelemetries()) + .hasSize(4); + } + + @Test + public void send_server_id_and_version() { + String id = randomAlphanumeric(40); + String version = randomAlphanumeric(10); + server.setId(id); + server.setVersion(version); + + TelemetryData data = communityUnderTest.load(); + assertThat(data.getServerId()).isEqualTo(id); + assertThat(data.getVersion()).isEqualTo(version); + + data = commercialUnderTest.load(); + assertThat(data.getServerId()).isEqualTo(id); + assertThat(data.getVersion()).isEqualTo(version); + } + + @Test + public void send_server_installation_date_and_installation_version() { + String installationVersion = "7.9.BEST.LTS.EVER"; + Long installationDate = 1546300800000L; // 2019/01/01 + internalProperties.write(InternalProperties.INSTALLATION_DATE, String.valueOf(installationDate)); + internalProperties.write(InternalProperties.INSTALLATION_VERSION, installationVersion); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.getInstallationDate()).isEqualTo(installationDate); + assertThat(data.getInstallationVersion()).isEqualTo(installationVersion); + } + + @Test + public void send_correct_sequence_number() { + internalProperties.write(TelemetryDaemon.I_PROP_MESSAGE_SEQUENCE, "10"); + TelemetryData data = communityUnderTest.load(); + assertThat(data.getMessageSequenceNumber()).isEqualTo(11L); + } + + @Test + public void do_not_send_server_installation_details_if_missing_property() { + TelemetryData data = communityUnderTest.load(); + assertThat(data.getInstallationDate()).isNull(); + assertThat(data.getInstallationVersion()).isNull(); + + data = commercialUnderTest.load(); + assertThat(data.getInstallationDate()).isNull(); + assertThat(data.getInstallationVersion()).isNull(); + } + + @Test + public void send_unanalyzed_languages_flags_when_edition_is_community() { + when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY)); + MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY)); + MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY)); + ComponentDto project1 = db.components().insertPublicProject().getMainBranchComponent(); + ComponentDto project2 = db.components().insertPublicProject().getMainBranchComponent(); + db.measures().insertLiveMeasure(project1, unanalyzedC); + db.measures().insertLiveMeasure(project2, unanalyzedC); + db.measures().insertLiveMeasure(project2, unanalyzedCpp); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.hasUnanalyzedC().get()).isTrue(); + assertThat(data.hasUnanalyzedCpp().get()).isTrue(); + } + + @Test + public void do_not_send_unanalyzed_languages_flags_when_edition_is_not_community() { + when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER)); + MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY)); + MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY)); + ComponentDto project1 = db.components().insertPublicProject().getMainBranchComponent(); + ComponentDto project2 = db.components().insertPublicProject().getMainBranchComponent(); + db.measures().insertLiveMeasure(project1, unanalyzedC); + db.measures().insertLiveMeasure(project2, unanalyzedCpp); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.hasUnanalyzedC()).isEmpty(); + assertThat(data.hasUnanalyzedCpp()).isEmpty(); + } + + @Test + public void unanalyzed_languages_flags_are_set_to_false_when_no_unanalyzed_languages_and_edition_is_community() { + when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY)); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.hasUnanalyzedC().get()).isFalse(); + assertThat(data.hasUnanalyzedCpp().get()).isFalse(); + } + + @Test + public void populate_security_custom_config_for_languages_on_enterprise() { + when(editionProvider.get()).thenReturn(Optional.of(ENTERPRISE)); + + when(configuration.get("sonar.security.config.javasecurity")).thenReturn(Optional.of("{}")); + when(configuration.get("sonar.security.config.phpsecurity")).thenReturn(Optional.of("{}")); + when(configuration.get("sonar.security.config.pythonsecurity")).thenReturn(Optional.of("{}")); + when(configuration.get("sonar.security.config.roslyn.sonaranalyzer.security.cs")).thenReturn(Optional.of("{}")); + + TelemetryData data = commercialUnderTest.load(); + + assertThat(data.getCustomSecurityConfigs()) + .containsExactlyInAnyOrder("java", "php", "python", "csharp"); + } + + @Test + public void skip_security_custom_config_on_community() { + when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY)); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.getCustomSecurityConfigs()).isEmpty(); + } + + @Test + public void undetected_alm_ci_slm_data() { + server.setId(SERVER_ID).setVersion("7.5.4"); + db.components().insertPublicProject().getMainBranchComponent(); + TelemetryData data = communityUnderTest.load(); + assertThat(data.getProjectStatistics()) + .extracting(ProjectStatistics::getDevopsPlatform, ProjectStatistics::getScm, ProjectStatistics::getCi) + .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected")); + } + + @Test + public void givenExistingExternalSecurityReport_whenTelemetryIsGenerated_payloadShouldContainLastUsageDate() { + server.setId(SERVER_ID).setVersion("7.5.4"); + ProjectData projectData = db.components().insertPublicProject(); + db.getDbClient().propertiesDao().saveProperty(new PropertyDto().setKey(EXTERNAL_SECURITY_REPORT_EXPORTED_AT).setEntityUuid(projectData.projectUuid()).setValue("1")); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.getProjectStatistics()).isNotEmpty(); + assertThat(data.getProjectStatistics().get(0).getExternalSecurityReportExportedAt()).isPresent() + .get().isEqualTo(1L); + } + + @Test + @UseDataProvider("getManagedInstanceData") + public void managedInstanceData_containsCorrectInformation(boolean isManaged, String provider) { + when(managedInstanceService.isInstanceExternallyManaged()).thenReturn(isManaged); + when(managedInstanceService.getProviderName()).thenReturn(provider); + + TelemetryData data = commercialUnderTest.load(); + + TelemetryData.ManagedInstanceInformation managedInstance = data.getManagedInstanceInformation(); + assertThat(managedInstance.isManaged()).isEqualTo(isManaged); + assertThat(managedInstance.provider()).isEqualTo(provider); + } + + @Test + public void load_shouldContainCloudUsage() { + CloudUsage cloudUsage = new CloudUsage(true, "1.27", "linux/amd64", "5.4.181-99.354.amzn2.x86_64", "10.1.0", "docker", false); + when(cloudUsageDataProvider.getCloudUsage()).thenReturn(cloudUsage); + + TelemetryData data = commercialUnderTest.load(); + assertThat(data.getCloudUsage()).isEqualTo(cloudUsage); + } + + @Test + public void default_quality_gate_sent_with_project() { + db.components().insertPublicProject().getMainBranchComponent(); + QualityGateDto qualityGate = db.qualityGates().insertQualityGate(qg -> qg.setName("anything").setBuiltIn(true)); + db.qualityGates().setDefaultQualityGate(qualityGate); + TelemetryData data = communityUnderTest.load(); + assertThat(data.getProjectStatistics()) + .extracting(ProjectStatistics::getQualityGate) + .containsOnly(qualityGate.getUuid()); + } + + private PluginInfo newPlugin(String key, String version) { + return new PluginInfo(key) + .setVersion(Version.create(version)); + } + + private void insertAnalysisProperty(SnapshotDto snapshotDto, String uuid, String key, String value) { + db.getDbClient().analysisPropertiesDao().insert(db.getSession(), new AnalysisPropertyDto() + .setUuid(uuid) + .setAnalysisUuid(snapshotDto.getUuid()) + .setKey(key) + .setValue(value) + .setCreatedAt(1L)); + } + + @DataProvider + public static Set<String> getScimFeatureStatues() { + HashSet<String> result = new HashSet<>(); + result.add("true"); + result.add("false"); + result.add(null); + return result; + } + + @DataProvider + public static Object[][] getManagedInstanceData() { + return new Object[][] { + {true, "scim"}, + {true, "github"}, + {true, "gitlab"}, + {false, null}, + }; + } +} diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/CloudUsageDataProvider.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/CloudUsageDataProvider.java new file mode 100644 index 00000000000..ac4c6cda698 --- /dev/null +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/CloudUsageDataProvider.java @@ -0,0 +1,236 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import com.google.common.annotations.VisibleForTesting; +import com.google.gson.Gson; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.SecureRandom; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.util.Collection; +import java.util.Scanner; +import java.util.function.Supplier; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okhttp3.internal.tls.OkHostnameVerifier; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.server.ServerSide; +import org.sonar.api.utils.System2; +import org.sonar.server.platform.ContainerSupport; +import org.sonar.server.util.Paths2; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +@ServerSide +public class CloudUsageDataProvider { + + private static final Logger LOG = LoggerFactory.getLogger(CloudUsageDataProvider.class); + + private static final String SERVICEACCOUNT_CA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; + static final String KUBERNETES_SERVICE_HOST = "KUBERNETES_SERVICE_HOST"; + static final String KUBERNETES_SERVICE_PORT = "KUBERNETES_SERVICE_PORT"; + static final String SONAR_HELM_CHART_VERSION = "SONAR_HELM_CHART_VERSION"; + static final String DOCKER_RUNNING = "DOCKER_RUNNING"; + private static final String[] KUBERNETES_PROVIDER_COMMAND = {"bash", "-c", "uname -r"}; + private static final int KUBERNETES_PROVIDER_MAX_SIZE = 100; + private final ContainerSupport containerSupport; + private final System2 system2; + private final Paths2 paths2; + private final Supplier<ProcessBuilder> processBuilderSupplier; + private OkHttpClient httpClient; + private TelemetryData.CloudUsage cloudUsageData; + + @Inject + public CloudUsageDataProvider(ContainerSupport containerSupport, System2 system2, Paths2 paths2) { + this(containerSupport, system2, paths2, ProcessBuilder::new, null); + if (isOnKubernetes()) { + initHttpClient(); + } + } + + @VisibleForTesting + CloudUsageDataProvider(ContainerSupport containerSupport, System2 system2, Paths2 paths2, Supplier<ProcessBuilder> processBuilderSupplier, + @Nullable OkHttpClient httpClient) { + this.containerSupport = containerSupport; + this.system2 = system2; + this.paths2 = paths2; + this.processBuilderSupplier = processBuilderSupplier; + this.httpClient = httpClient; + } + + public TelemetryData.CloudUsage getCloudUsage() { + if (cloudUsageData != null) { + return cloudUsageData; + } + + String kubernetesVersion = null; + String kubernetesPlatform = null; + + if (isOnKubernetes()) { + VersionInfo versionInfo = getVersionInfo(); + if (versionInfo != null) { + kubernetesVersion = versionInfo.major() + "." + versionInfo.minor(); + kubernetesPlatform = versionInfo.platform(); + } + } + + cloudUsageData = new TelemetryData.CloudUsage( + isOnKubernetes(), + kubernetesVersion, + kubernetesPlatform, + getKubernetesProvider(), + getOfficialHelmChartVersion(), + containerSupport.getContainerContext(), + isOfficialImageUsed()); + + return cloudUsageData; + } + + private boolean isOnKubernetes() { + return StringUtils.isNotBlank(system2.envVariable(KUBERNETES_SERVICE_HOST)); + } + + @CheckForNull + private String getOfficialHelmChartVersion() { + return system2.envVariable(SONAR_HELM_CHART_VERSION); + } + + private boolean isOfficialImageUsed() { + return Boolean.parseBoolean(system2.envVariable(DOCKER_RUNNING)); + } + + /** + * Create an http client to call the Kubernetes API. + * This is based on the client creation in the official Kubernetes Java client. + */ + private void initHttpClient() { + try { + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(getKeyStore()); + TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagers, new SecureRandom()); + + httpClient = new OkHttpClient.Builder() + .sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]) + .hostnameVerifier(OkHostnameVerifier.INSTANCE) + .build(); + } catch (Exception e) { + LOG.debug("Failed to create http client for Kubernetes API", e); + } + } + + private KeyStore getKeyStore() throws GeneralSecurityException, IOException { + KeyStore caKeyStore = newEmptyKeyStore(); + + try (FileInputStream fis = new FileInputStream(paths2.get(SERVICEACCOUNT_CA_PATH).toFile())) { + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + Collection<? extends Certificate> certificates = certificateFactory.generateCertificates(fis); + + int index = 0; + for (Certificate certificate : certificates) { + String certificateAlias = "ca" + index; + caKeyStore.setCertificateEntry(certificateAlias, certificate); + index++; + } + } + + return caKeyStore; + } + + private static KeyStore newEmptyKeyStore() throws GeneralSecurityException, IOException { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null, null); + return keyStore; + } + + record VersionInfo(String major, String minor, String platform) { + } + + private VersionInfo getVersionInfo() { + try { + Request request = buildRequest(); + try (Response response = httpClient.newCall(request).execute()) { + ResponseBody responseBody = requireNonNull(response.body(), "Response body is null"); + return new Gson().fromJson(responseBody.string(), VersionInfo.class); + } + } catch (Exception e) { + LOG.debug("Failed to get Kubernetes version info", e); + return null; + } + } + + private Request buildRequest() throws URISyntaxException { + String host = system2.envVariable(KUBERNETES_SERVICE_HOST); + String port = system2.envVariable(KUBERNETES_SERVICE_PORT); + if (host == null || port == null) { + throw new IllegalStateException("Kubernetes environment variables are not set"); + } + + URI uri = new URI("https", null, host, Integer.parseInt(port), "/version", null, null); + + return new Request.Builder() + .get() + .url(uri.toString()) + .build(); + } + + @CheckForNull + private String getKubernetesProvider() { + try { + Process process = processBuilderSupplier.get().command(KUBERNETES_PROVIDER_COMMAND).start(); + try (Scanner scanner = new Scanner(process.getInputStream(), UTF_8)) { + scanner.useDelimiter("\n"); + // Null characters can be present in the output on Windows + String output = scanner.next().replace("\u0000", ""); + return StringUtils.abbreviate(output, KUBERNETES_PROVIDER_MAX_SIZE); + } finally { + process.destroy(); + } + } catch (Exception e) { + LOG.debug("Failed to get Kubernetes provider", e); + return null; + } + } + + @VisibleForTesting + OkHttpClient getHttpClient() { + return httpClient; + } +} diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/QualityProfileDataProvider.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/QualityProfileDataProvider.java new file mode 100644 index 00000000000..fd0747529e3 --- /dev/null +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/QualityProfileDataProvider.java @@ -0,0 +1,95 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.qualityprofile.QProfileDto; +import org.sonar.server.qualityprofile.QProfileComparison; + +import static java.util.stream.Collectors.toMap; + +public class QualityProfileDataProvider { + + private final DbClient dbClient; + private final QProfileComparison qProfileComparison; + + public QualityProfileDataProvider(DbClient dbClient, QProfileComparison qProfileComparison) { + this.dbClient = dbClient; + this.qProfileComparison = qProfileComparison; + } + + public List<TelemetryData.QualityProfile> retrieveQualityProfilesData() { + try (DbSession dbSession = dbClient.openSession(false)) { + + Set<String> defaultProfileUuids = dbClient.qualityProfileDao().selectAllDefaultProfiles(dbSession) + .stream().map(QProfileDto::getKee) + .collect(Collectors.toSet()); + + Map<String, QProfileDto> allProfileDtosByUuid = dbClient.qualityProfileDao().selectAll(dbSession) + .stream() + .collect(toMap(QProfileDto::getKee, p -> p)); + + return allProfileDtosByUuid.entrySet().stream() + .map(p -> mapQualityProfile(p.getValue(), allProfileDtosByUuid, defaultProfileUuids.contains(p.getKey()), dbSession)) + .toList(); + } + } + + private TelemetryData.QualityProfile mapQualityProfile(QProfileDto profile, Map<String, QProfileDto> allProfileDtos, boolean isDefault, DbSession dbSession) { + QProfileDto rootProfile = getRootProfile(profile.getKee(), allProfileDtos); + Boolean isBuiltInRootParent; + if (profile.isBuiltIn()) { + isBuiltInRootParent = null; + } else { + isBuiltInRootParent = rootProfile.isBuiltIn() && !rootProfile.getKee().equals(profile.getKee()); + } + + Optional<QProfileComparison.QProfileComparisonResult> rulesComparison = Optional.of(profile) + .filter(p -> isBuiltInRootParent != null && isBuiltInRootParent) + .map(p -> qProfileComparison.compare(dbSession, rootProfile, profile)); + + return new TelemetryData.QualityProfile(profile.getKee(), + profile.getParentKee(), + profile.getLanguage(), + isDefault, + profile.isBuiltIn(), + isBuiltInRootParent, + rulesComparison.map(c -> c.modified().size()).orElse(null), + rulesComparison.map(c -> c.inRight().size()).orElse(null), + rulesComparison.map(c -> c.inLeft().size()).orElse(null) + ); + } + + public QProfileDto getRootProfile(String kee, Map<String, QProfileDto> allProfileDtos) { + QProfileDto qProfileDto = allProfileDtos.get(kee); + String parentKee = qProfileDto.getParentKee(); + if (parentKee != null) { + return getRootProfile(parentKee, allProfileDtos); + } else { + return allProfileDtos.get(kee); + } + } +} diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryClient.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryClient.java new file mode 100644 index 00000000000..20d792aff4b --- /dev/null +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryClient.java @@ -0,0 +1,127 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import java.io.IOException; +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okio.BufferedSink; +import okio.GzipSink; +import okio.Okio; +import org.sonar.api.Startable; +import org.sonar.api.config.Configuration; +import org.sonar.api.server.ServerSide; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_COMPRESSION; +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_URL; + +@ServerSide +public class TelemetryClient implements Startable { + private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); + private static final Logger LOG = LoggerFactory.getLogger(TelemetryClient.class); + + private final OkHttpClient okHttpClient; + private final Configuration config; + private String serverUrl; + private boolean compression; + + public TelemetryClient(OkHttpClient okHttpClient, Configuration config) { + this.config = config; + this.okHttpClient = okHttpClient; + } + + void upload(String json) throws IOException { + Request request = buildHttpRequest(json); + execute(okHttpClient.newCall(request)); + } + + void optOut(String json) { + Request.Builder request = new Request.Builder(); + request.url(serverUrl); + RequestBody body = RequestBody.create(JSON, json); + request.delete(body); + + try { + execute(okHttpClient.newCall(request.build())); + } catch (IOException e) { + LOG.debug("Error when sending opt-out usage statistics: {}", e.getMessage()); + } + } + + private Request buildHttpRequest(String json) { + Request.Builder request = new Request.Builder(); + request.addHeader("Content-Encoding", "gzip"); + request.addHeader("Content-Type", "application/json"); + request.url(serverUrl); + RequestBody body = RequestBody.create(JSON, json); + if (compression) { + request.post(gzip(body)); + } else { + request.post(body); + } + return request.build(); + } + + private static RequestBody gzip(final RequestBody body) { + return new RequestBody() { + @Override + public MediaType contentType() { + return body.contentType(); + } + + @Override + public long contentLength() { + // We don't know the compressed length in advance! + return -1; + } + + @Override + public void writeTo(BufferedSink sink) throws IOException { + BufferedSink gzipSink = Okio.buffer(new GzipSink(sink)); + body.writeTo(gzipSink); + gzipSink.close(); + } + }; + } + + private static void execute(Call call) throws IOException { + try (Response ignored = call.execute()) { + // auto close connection to avoid leaked connection + } + } + + @Override + public void start() { + this.serverUrl = config.get(SONAR_TELEMETRY_URL.getKey()) + .orElseThrow(() -> new IllegalStateException(String.format("Setting '%s' must be provided.", SONAR_TELEMETRY_URL))); + this.compression = config.getBoolean(SONAR_TELEMETRY_COMPRESSION.getKey()).orElse(true); + } + + @Override + public void stop() { + // Nothing to do + } +} diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDaemon.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDaemon.java new file mode 100644 index 00000000000..a65a1d9ccd9 --- /dev/null +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDaemon.java @@ -0,0 +1,169 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.io.IOException; +import java.io.StringWriter; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import org.sonar.api.config.Configuration; +import org.sonar.api.server.ServerSide; +import org.sonar.api.utils.System2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.utils.text.JsonWriter; +import org.sonar.server.property.InternalProperties; +import org.sonar.server.util.AbstractStoppableScheduledExecutorServiceImpl; +import org.sonar.server.util.GlobalLockManager; + +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_ENABLE; +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_FREQUENCY_IN_SECONDS; +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_URL; + +@ServerSide +public class TelemetryDaemon extends AbstractStoppableScheduledExecutorServiceImpl<ScheduledExecutorService> { + private static final String THREAD_NAME_PREFIX = "sq-telemetry-service-"; + private static final int ONE_DAY = 24 * 60 * 60 * 1_000; + private static final String I_PROP_LAST_PING = "telemetry.lastPing"; + private static final String I_PROP_OPT_OUT = "telemetry.optOut"; + private static final String LOCK_NAME = "TelemetryStat"; + private static final Logger LOG = LoggerFactory.getLogger(TelemetryDaemon.class); + private static final String LOCK_DELAY_SEC = "sonar.telemetry.lock.delay"; + static final String I_PROP_MESSAGE_SEQUENCE = "telemetry.messageSeq"; + + private final TelemetryDataLoader dataLoader; + private final TelemetryDataJsonWriter dataJsonWriter; + private final TelemetryClient telemetryClient; + private final GlobalLockManager lockManager; + private final Configuration config; + private final InternalProperties internalProperties; + private final System2 system2; + + public TelemetryDaemon(TelemetryDataLoader dataLoader, TelemetryDataJsonWriter dataJsonWriter, TelemetryClient telemetryClient, Configuration config, + InternalProperties internalProperties, GlobalLockManager lockManager, System2 system2) { + super(Executors.newSingleThreadScheduledExecutor(newThreadFactory())); + this.dataLoader = dataLoader; + this.dataJsonWriter = dataJsonWriter; + this.telemetryClient = telemetryClient; + this.config = config; + this.internalProperties = internalProperties; + this.lockManager = lockManager; + this.system2 = system2; + } + + @Override + public void start() { + boolean isTelemetryActivated = config.getBoolean(SONAR_TELEMETRY_ENABLE.getKey()) + .orElseThrow(() -> new IllegalStateException(String.format("Setting '%s' must be provided.", SONAR_TELEMETRY_URL.getKey()))); + boolean hasOptOut = internalProperties.read(I_PROP_OPT_OUT).isPresent(); + if (!isTelemetryActivated && !hasOptOut) { + optOut(); + internalProperties.write(I_PROP_OPT_OUT, String.valueOf(system2.now())); + LOG.info("Sharing of SonarQube statistics is disabled."); + } + if (isTelemetryActivated && hasOptOut) { + internalProperties.write(I_PROP_OPT_OUT, null); + } + if (!isTelemetryActivated) { + return; + } + LOG.info("Sharing of SonarQube statistics is enabled."); + int frequencyInSeconds = frequency(); + scheduleWithFixedDelay(telemetryCommand(), frequencyInSeconds, frequencyInSeconds, TimeUnit.SECONDS); + } + + private static ThreadFactory newThreadFactory() { + return new ThreadFactoryBuilder() + .setNameFormat(THREAD_NAME_PREFIX + "%d") + .setPriority(Thread.MIN_PRIORITY) + .build(); + } + + private Runnable telemetryCommand() { + return () -> { + try { + + if (!lockManager.tryLock(LOCK_NAME, lockDuration())) { + return; + } + + long now = system2.now(); + if (shouldUploadStatistics(now)) { + uploadStatistics(); + updateTelemetryProps(now); + } + } catch (Exception e) { + LOG.debug("Error while checking SonarQube statistics: {}", e.getMessage(), e); + } + // do not check at start up to exclude test instance which are not up for a long time + }; + } + + private void updateTelemetryProps(long now) { + internalProperties.write(I_PROP_LAST_PING, String.valueOf(now)); + + Optional<String> currentSequence = internalProperties.read(I_PROP_MESSAGE_SEQUENCE); + if (currentSequence.isEmpty()) { + internalProperties.write(I_PROP_MESSAGE_SEQUENCE, String.valueOf(1)); + return; + } + + long current = Long.parseLong(currentSequence.get()); + internalProperties.write(I_PROP_MESSAGE_SEQUENCE, String.valueOf(current + 1)); + } + + private void optOut() { + StringWriter json = new StringWriter(); + try (JsonWriter writer = JsonWriter.of(json)) { + writer.beginObject(); + writer.prop("id", dataLoader.loadServerId()); + writer.endObject(); + } + telemetryClient.optOut(json.toString()); + } + + private void uploadStatistics() throws IOException { + TelemetryData statistics = dataLoader.load(); + StringWriter jsonString = new StringWriter(); + try (JsonWriter json = JsonWriter.of(jsonString)) { + dataJsonWriter.writeTelemetryData(json, statistics); + } + telemetryClient.upload(jsonString.toString()); + dataLoader.reset(); + } + + private boolean shouldUploadStatistics(long now) { + Optional<Long> lastPing = internalProperties.read(I_PROP_LAST_PING).map(Long::valueOf); + return lastPing.isEmpty() || now - lastPing.get() >= ONE_DAY; + } + + private int frequency() { + return config.getInt(SONAR_TELEMETRY_FREQUENCY_IN_SECONDS.getKey()) + .orElseThrow(() -> new IllegalStateException(String.format("Setting '%s' must be provided.", SONAR_TELEMETRY_FREQUENCY_IN_SECONDS))); + } + + private int lockDuration() { + return config.getInt(LOCK_DELAY_SEC).orElse(60); + } +} diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryData.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryData.java new file mode 100644 index 00000000000..28c270fb3c9 --- /dev/null +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryData.java @@ -0,0 +1,596 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import javax.annotation.Nullable; +import org.sonar.core.platform.EditionProvider.Edition; +import org.sonar.db.project.CreationMethod; +import org.sonar.db.user.UserTelemetryDto; +import org.sonar.server.qualitygate.Condition; + +import static java.util.Objects.requireNonNullElse; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.PREVIOUS_VERSION; + +public class TelemetryData { + private final String serverId; + private final String version; + private final Long messageSequenceNumber; + private final Map<String, String> plugins; + private final Database database; + private final Edition edition; + private final String defaultQualityGate; + private final String sonarWayQualityGate; + private final Long installationDate; + private final String installationVersion; + private final boolean inContainer; + private final ManagedInstanceInformation managedInstanceInformation; + private final CloudUsage cloudUsage; + private final List<UserTelemetryDto> users; + private final List<Project> projects; + private final List<ProjectStatistics> projectStatistics; + private final List<Branch> branches; + private final List<QualityGate> qualityGates; + private final List<QualityProfile> qualityProfiles; + private final Collection<NewCodeDefinition> newCodeDefinitions; + private final Boolean hasUnanalyzedC; + private final Boolean hasUnanalyzedCpp; + private final int ncdId; + private final Set<String> customSecurityConfigs; + + private TelemetryData(Builder builder) { + serverId = builder.serverId; + version = builder.version; + messageSequenceNumber = builder.messageSequenceNumber; + plugins = builder.plugins; + database = builder.database; + edition = builder.edition; + defaultQualityGate = builder.defaultQualityGate; + sonarWayQualityGate = builder.sonarWayQualityGate; + installationDate = builder.installationDate; + installationVersion = builder.installationVersion; + inContainer = builder.inContainer; + users = builder.users; + projects = builder.projects; + projectStatistics = builder.projectStatistics; + qualityGates = builder.qualityGates; + qualityProfiles = builder.qualityProfiles; + hasUnanalyzedC = builder.hasUnanalyzedC; + hasUnanalyzedCpp = builder.hasUnanalyzedCpp; + customSecurityConfigs = requireNonNullElse(builder.customSecurityConfigs, Set.of()); + managedInstanceInformation = builder.managedInstanceInformation; + cloudUsage = builder.cloudUsage; + ncdId = builder.ncdId; + branches = builder.branches; + newCodeDefinitions = builder.newCodeDefinitions; + } + + public String getServerId() { + return serverId; + } + + public String getVersion() { + return version; + } + + public Long getMessageSequenceNumber() { + return messageSequenceNumber; + } + + public Map<String, String> getPlugins() { + return plugins; + } + + public Database getDatabase() { + return database; + } + + public Optional<Edition> getEdition() { + return Optional.ofNullable(edition); + } + + public String getDefaultQualityGate() { + return defaultQualityGate; + } + + public String getSonarWayQualityGate() { + return sonarWayQualityGate; + } + + public Long getInstallationDate() { + return installationDate; + } + + public String getInstallationVersion() { + return installationVersion; + } + + public boolean isInContainer() { + return inContainer; + } + + public ManagedInstanceInformation getManagedInstanceInformation() { + return managedInstanceInformation; + } + + public CloudUsage getCloudUsage() { + return cloudUsage; + } + + public Optional<Boolean> hasUnanalyzedC() { + return Optional.ofNullable(hasUnanalyzedC); + } + + public Optional<Boolean> hasUnanalyzedCpp() { + return Optional.ofNullable(hasUnanalyzedCpp); + } + + public Set<String> getCustomSecurityConfigs() { + return customSecurityConfigs; + } + + public List<UserTelemetryDto> getUserTelemetries() { + return users; + } + + public List<Project> getProjects() { + return projects; + } + + public List<ProjectStatistics> getProjectStatistics() { + return projectStatistics; + } + + public List<QualityGate> getQualityGates() { + return qualityGates; + } + + public List<QualityProfile> getQualityProfiles() { + return qualityProfiles; + } + + static Builder builder() { + return new Builder(); + } + + public int getNcdId() { + return ncdId; + } + + public List<Branch> getBranches() { + return branches; + } + + public Collection<NewCodeDefinition> getNewCodeDefinitions() { + return newCodeDefinitions; + } + + static class Builder { + private String serverId; + private String version; + private Long messageSequenceNumber; + private Map<String, String> plugins; + private Database database; + private Edition edition; + private String defaultQualityGate; + + private String sonarWayQualityGate; + private Long installationDate; + private String installationVersion; + private boolean inContainer = false; + private ManagedInstanceInformation managedInstanceInformation; + private CloudUsage cloudUsage; + private Boolean hasUnanalyzedC; + private Boolean hasUnanalyzedCpp; + private Set<String> customSecurityConfigs; + private List<UserTelemetryDto> users; + private List<Project> projects; + private List<ProjectStatistics> projectStatistics; + private List<Branch> branches; + private Collection<NewCodeDefinition> newCodeDefinitions; + private List<QualityGate> qualityGates; + private List<QualityProfile> qualityProfiles; + private int ncdId; + + private Builder() { + // enforce static factory method + } + + Builder setServerId(String serverId) { + this.serverId = serverId; + return this; + } + + Builder setVersion(String version) { + this.version = version; + return this; + } + + Builder setMessageSequenceNumber(@Nullable Long messageSequenceNumber) { + this.messageSequenceNumber = messageSequenceNumber; + return this; + } + + Builder setPlugins(Map<String, String> plugins) { + this.plugins = plugins; + return this; + } + + Builder setDatabase(Database database) { + this.database = database; + return this; + } + + Builder setEdition(@Nullable Edition edition) { + this.edition = edition; + return this; + } + + Builder setDefaultQualityGate(String defaultQualityGate) { + this.defaultQualityGate = defaultQualityGate; + return this; + } + + Builder setSonarWayQualityGate(String sonarWayQualityGate) { + this.sonarWayQualityGate = sonarWayQualityGate; + return this; + } + + Builder setInstallationDate(@Nullable Long installationDate) { + this.installationDate = installationDate; + return this; + } + + Builder setInstallationVersion(@Nullable String installationVersion) { + this.installationVersion = installationVersion; + return this; + } + + Builder setInContainer(boolean inContainer) { + this.inContainer = inContainer; + return this; + } + + Builder setHasUnanalyzedC(@Nullable Boolean hasUnanalyzedC) { + this.hasUnanalyzedC = hasUnanalyzedC; + return this; + } + + Builder setHasUnanalyzedCpp(@Nullable Boolean hasUnanalyzedCpp) { + this.hasUnanalyzedCpp = hasUnanalyzedCpp; + return this; + } + + Builder setCustomSecurityConfigs(Set<String> customSecurityConfigs) { + this.customSecurityConfigs = customSecurityConfigs; + return this; + } + + Builder setUsers(List<UserTelemetryDto> users) { + this.users = users; + return this; + } + + Builder setProjects(List<Project> projects) { + this.projects = projects; + return this; + } + + Builder setManagedInstanceInformation(ManagedInstanceInformation managedInstanceInformation) { + this.managedInstanceInformation = managedInstanceInformation; + return this; + } + + Builder setCloudUsage(CloudUsage cloudUsage) { + this.cloudUsage = cloudUsage; + return this; + } + + TelemetryData build() { + requireNonNullValues(serverId, version, plugins, database, messageSequenceNumber); + return new TelemetryData(this); + } + + Builder setProjectStatistics(List<ProjectStatistics> projectStatistics) { + this.projectStatistics = projectStatistics; + return this; + } + + Builder setQualityGates(List<QualityGate> qualityGates) { + this.qualityGates = qualityGates; + return this; + } + + Builder setQualityProfiles(List<QualityProfile> qualityProfiles) { + this.qualityProfiles = qualityProfiles; + return this; + } + + Builder setNcdId(int ncdId) { + this.ncdId = ncdId; + return this; + } + + private static void requireNonNullValues(Object... values) { + Arrays.stream(values).forEach(Objects::requireNonNull); + } + + Builder setBranches(List<Branch> branches) { + this.branches = branches; + return this; + } + + Builder setNewCodeDefinitions(Collection<NewCodeDefinition> newCodeDefinitions) { + this.newCodeDefinitions = newCodeDefinitions; + return this; + } + } + + record Database(String name, String version) { + } + + record NewCodeDefinition(String type, @Nullable String value, String scope) { + + private static final NewCodeDefinition instanceDefault = new NewCodeDefinition(PREVIOUS_VERSION.name(), "", "instance"); + + public static NewCodeDefinition getInstanceDefault() { + return instanceDefault; + } + + @Override + public String value() { + return value == null ? "" : value; + } + } + + record Branch(String projectUuid, String branchUuid, int ncdId, int greenQualityGateCount, int analysisCount, boolean excludeFromPurge) { + } + + record Project(String projectUuid, Long lastAnalysis, String language, String qualityProfile, Long loc) { + } + + record QualityGate(String uuid, String caycStatus, List<Condition> conditions) { + } + + public record QualityProfile(String uuid, @Nullable String parentUuid, String language, boolean isDefault, + boolean isBuiltIn, + @Nullable Boolean builtInParent, @Nullable Integer rulesOverriddenCount, + @Nullable Integer rulesActivatedCount, @Nullable Integer rulesDeactivatedCount) { + } + + record ManagedInstanceInformation(boolean isManaged, @Nullable String provider) { + } + + record CloudUsage(boolean kubernetes, @Nullable String kubernetesVersion, @Nullable String kubernetesPlatform, + @Nullable String kubernetesProvider, + @Nullable String officialHelmChart, @Nullable String containerRuntime, boolean officialImage) { + } + + public static class ProjectStatistics { + private final String projectUuid; + private final Long branchCount; + private final Long pullRequestCount; + private final String qualityGate; + private final String scm; + private final String ci; + private final String devopsPlatform; + private final Long bugs; + private final Long vulnerabilities; + private final Long securityHotspots; + private final Long technicalDebt; + private final Long developmentCost; + private final int ncdId; + private final Long externalSecurityReportExportedAt; + private final CreationMethod creationMethod; + private final Boolean monorepo; + + ProjectStatistics(Builder builder) { + this.projectUuid = builder.projectUuid; + this.branchCount = builder.branchCount; + this.pullRequestCount = builder.pullRequestCount; + this.qualityGate = builder.qualityGate; + this.scm = builder.scm; + this.ci = builder.ci; + this.devopsPlatform = builder.devopsPlatform; + this.bugs = builder.bugs; + this.vulnerabilities = builder.vulnerabilities; + this.securityHotspots = builder.securityHotspots; + this.technicalDebt = builder.technicalDebt; + this.developmentCost = builder.developmentCost; + this.ncdId = builder.ncdId; + this.externalSecurityReportExportedAt = builder.externalSecurityReportExportedAt; + this.creationMethod = builder.creationMethod; + this.monorepo = builder.monorepo; + } + + public int getNcdId() { + return ncdId; + } + + public String getProjectUuid() { + return projectUuid; + } + + public Long getBranchCount() { + return branchCount; + } + + public Long getPullRequestCount() { + return pullRequestCount; + } + + public String getQualityGate() { + return qualityGate; + } + + public String getScm() { + return scm; + } + + public String getCi() { + return ci; + } + + public String getDevopsPlatform() { + return devopsPlatform; + } + + public Optional<Long> getBugs() { + return Optional.ofNullable(bugs); + } + + public Optional<Long> getVulnerabilities() { + return Optional.ofNullable(vulnerabilities); + } + + public Optional<Long> getSecurityHotspots() { + return Optional.ofNullable(securityHotspots); + } + + public Optional<Long> getTechnicalDebt() { + return Optional.ofNullable(technicalDebt); + } + + public Optional<Long> getDevelopmentCost() { + return Optional.ofNullable(developmentCost); + } + + public Optional<Long> getExternalSecurityReportExportedAt() { + return Optional.ofNullable(externalSecurityReportExportedAt); + } + + public CreationMethod getCreationMethod() { + return creationMethod; + } + + public Boolean isMonorepo() { + return monorepo; + } + + static class Builder { + private String projectUuid; + private Long branchCount; + private Long pullRequestCount; + private String qualityGate; + private String scm; + private String ci; + private String devopsPlatform; + private Long bugs; + private Long vulnerabilities; + private Long securityHotspots; + private Long technicalDebt; + private Long developmentCost; + private int ncdId; + private Long externalSecurityReportExportedAt; + private CreationMethod creationMethod; + private Boolean monorepo; + + public Builder setProjectUuid(String projectUuid) { + this.projectUuid = projectUuid; + return this; + } + + public Builder setNcdId(int ncdId) { + this.ncdId = ncdId; + return this; + } + + public Builder setBranchCount(Long branchCount) { + this.branchCount = branchCount; + return this; + } + + public Builder setPRCount(Long pullRequestCount) { + this.pullRequestCount = pullRequestCount; + return this; + } + + public Builder setQG(String qualityGate) { + this.qualityGate = qualityGate; + return this; + } + + public Builder setScm(String scm) { + this.scm = scm; + return this; + } + + public Builder setCi(String ci) { + this.ci = ci; + return this; + } + + public Builder setDevops(String devopsPlatform) { + this.devopsPlatform = devopsPlatform; + return this; + } + + public Builder setBugs(@Nullable Number bugs) { + this.bugs = bugs != null ? bugs.longValue() : null; + return this; + } + + public Builder setVulnerabilities(@Nullable Number vulnerabilities) { + this.vulnerabilities = vulnerabilities != null ? vulnerabilities.longValue() : null; + return this; + } + + public Builder setSecurityHotspots(@Nullable Number securityHotspots) { + this.securityHotspots = securityHotspots != null ? securityHotspots.longValue() : null; + return this; + } + + public Builder setTechnicalDebt(@Nullable Number technicalDebt) { + this.technicalDebt = technicalDebt != null ? technicalDebt.longValue() : null; + return this; + } + + public Builder setDevelopmentCost(@Nullable Number developmentCost) { + this.developmentCost = developmentCost != null ? developmentCost.longValue() : null; + return this; + } + + public Builder setExternalSecurityReportExportedAt(@Nullable Number externalSecurityReportExportedAt) { + this.externalSecurityReportExportedAt = externalSecurityReportExportedAt != null ? externalSecurityReportExportedAt.longValue() : null; + return this; + } + + public Builder setCreationMethod(CreationMethod creationMethod) { + this.creationMethod = creationMethod; + return this; + } + + public Builder setMonorepo(Boolean monorepo) { + this.monorepo = monorepo; + return this; + } + + public ProjectStatistics build() { + return new ProjectStatistics(this); + } + } + } +} diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDataJsonWriter.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDataJsonWriter.java new file mode 100644 index 00000000000..d2cb6cd231c --- /dev/null +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDataJsonWriter.java @@ -0,0 +1,293 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import com.google.common.annotations.VisibleForTesting; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Locale; +import org.jetbrains.annotations.NotNull; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.text.JsonWriter; +import org.sonar.core.telemetry.TelemetryExtension; +import org.sonar.server.util.DigestUtil; + +import static org.sonar.api.utils.DateUtils.DATETIME_FORMAT; + +public class TelemetryDataJsonWriter { + + @VisibleForTesting + static final String MANAGED_INSTANCE_PROPERTY = "managedInstanceInformation"; + @VisibleForTesting + static final String CLOUD_USAGE_PROPERTY = "cloudUsage"; + + private static final String LANGUAGE_PROPERTY = "language"; + private static final String VERSION = "version"; + private static final String NCD_ID = "ncdId"; + private static final String PROJECT_ID = "projectUuid"; + + private final List<TelemetryExtension> extensions; + + private final System2 system2; + + public TelemetryDataJsonWriter(List<TelemetryExtension> extensions, System2 system2) { + this.extensions = extensions; + this.system2 = system2; + } + + public void writeTelemetryData(JsonWriter json, TelemetryData telemetryData) { + json.beginObject(); + json.prop("id", telemetryData.getServerId()); + json.prop(VERSION, telemetryData.getVersion()); + json.prop("messageSequenceNumber", telemetryData.getMessageSequenceNumber()); + json.prop("localTimestamp", toUtc(system2.now())); + json.prop(NCD_ID, telemetryData.getNcdId()); + telemetryData.getEdition().ifPresent(e -> json.prop("edition", e.name().toLowerCase(Locale.ENGLISH))); + json.prop("defaultQualityGate", telemetryData.getDefaultQualityGate()); + json.prop("sonarway_quality_gate_uuid", telemetryData.getSonarWayQualityGate()); + json.name("database"); + json.beginObject(); + json.prop("name", telemetryData.getDatabase().name()); + json.prop(VERSION, telemetryData.getDatabase().version()); + json.endObject(); + json.name("plugins"); + json.beginArray(); + telemetryData.getPlugins().forEach((plugin, version) -> { + json.beginObject(); + json.prop("name", plugin); + json.prop(VERSION, version); + json.endObject(); + }); + json.endArray(); + + if (!telemetryData.getCustomSecurityConfigs().isEmpty()) { + json.name("customSecurityConfig"); + json.beginArray(); + json.values(telemetryData.getCustomSecurityConfigs()); + json.endArray(); + } + + telemetryData.hasUnanalyzedC().ifPresent(hasUnanalyzedC -> json.prop("hasUnanalyzedC", hasUnanalyzedC)); + telemetryData.hasUnanalyzedCpp().ifPresent(hasUnanalyzedCpp -> json.prop("hasUnanalyzedCpp", hasUnanalyzedCpp)); + + if (telemetryData.getInstallationDate() != null) { + json.prop("installationDate", toUtc(telemetryData.getInstallationDate())); + } + if (telemetryData.getInstallationVersion() != null) { + json.prop("installationVersion", telemetryData.getInstallationVersion()); + } + json.prop("container", telemetryData.isInContainer()); + + writeUserData(json, telemetryData); + writeProjectData(json, telemetryData); + writeProjectStatsData(json, telemetryData); + writeBranches(json, telemetryData); + writeNewCodeDefinitions(json, telemetryData); + writeQualityGates(json, telemetryData); + writeQualityProfiles(json, telemetryData); + writeManagedInstanceInformation(json, telemetryData.getManagedInstanceInformation()); + writeCloudUsage(json, telemetryData.getCloudUsage()); + extensions.forEach(e -> e.write(json)); + + json.endObject(); + } + + private static void writeUserData(JsonWriter json, TelemetryData telemetryData) { + if (telemetryData.getUserTelemetries() != null) { + json.name("users"); + json.beginArray(); + telemetryData.getUserTelemetries().forEach(user -> { + json.beginObject(); + json.prop("userUuid", DigestUtil.sha3_224Hex(user.getUuid())); + json.prop("status", user.isActive() ? "active" : "inactive"); + json.prop("identityProvider", user.getExternalIdentityProvider()); + + if (user.getLastConnectionDate() != null) { + json.prop("lastActivity", toUtc(user.getLastConnectionDate())); + } + if (user.getLastSonarlintConnectionDate() != null) { + json.prop("lastSonarlintActivity", toUtc(user.getLastSonarlintConnectionDate())); + } + json.prop("managed", user.getScimUuid() != null); + + json.endObject(); + }); + json.endArray(); + } + } + + private static void writeProjectData(JsonWriter json, TelemetryData telemetryData) { + if (telemetryData.getProjects() != null) { + json.name("projects"); + json.beginArray(); + telemetryData.getProjects().forEach(project -> { + json.beginObject(); + json.prop(PROJECT_ID, project.projectUuid()); + if (project.lastAnalysis() != null) { + json.prop("lastAnalysis", toUtc(project.lastAnalysis())); + } + json.prop(LANGUAGE_PROPERTY, project.language()); + json.prop("loc", project.loc()); + json.prop("qualityProfile", project.qualityProfile()); + json.endObject(); + }); + json.endArray(); + } + } + + private static void writeBranches(JsonWriter json, TelemetryData telemetryData) { + if (telemetryData.getBranches() != null) { + json.name("branches"); + json.beginArray(); + telemetryData.getBranches().forEach(branch -> { + json.beginObject(); + json.prop(PROJECT_ID, branch.projectUuid()); + json.prop("branchUuid", branch.branchUuid()); + json.prop(NCD_ID, branch.ncdId()); + json.prop("greenQualityGateCount", branch.greenQualityGateCount()); + json.prop("analysisCount", branch.analysisCount()); + json.prop("excludeFromPurge", branch.excludeFromPurge()); + json.endObject(); + }); + json.endArray(); + } + } + + private static void writeNewCodeDefinitions(JsonWriter json, TelemetryData telemetryData) { + if (telemetryData.getNewCodeDefinitions() != null) { + json.name("new-code-definitions"); + json.beginArray(); + telemetryData.getNewCodeDefinitions().forEach(ncd -> { + json.beginObject(); + json.prop(NCD_ID, ncd.hashCode()); + json.prop("type", ncd.type()); + json.prop("value", ncd.value()); + json.prop("scope", ncd.scope()); + json.endObject(); + }); + json.endArray(); + } + } + + private static void writeProjectStatsData(JsonWriter json, TelemetryData telemetryData) { + if (telemetryData.getProjectStatistics() != null) { + json.name("projects-general-stats"); + json.beginArray(); + telemetryData.getProjectStatistics().forEach(project -> { + json.beginObject(); + json.prop(PROJECT_ID, project.getProjectUuid()); + json.prop("branchCount", project.getBranchCount()); + json.prop("pullRequestCount", project.getPullRequestCount()); + json.prop("qualityGate", project.getQualityGate()); + json.prop("scm", project.getScm()); + json.prop("ci", project.getCi()); + json.prop("devopsPlatform", project.getDevopsPlatform()); + json.prop(NCD_ID, project.getNcdId()); + json.prop("project_creation_method", project.getCreationMethod().name()); + json.prop("monorepo", project.isMonorepo()); + project.getBugs().ifPresent(bugs -> json.prop("bugs", bugs)); + project.getVulnerabilities().ifPresent(vulnerabilities -> json.prop("vulnerabilities", vulnerabilities)); + project.getSecurityHotspots().ifPresent(securityHotspots -> json.prop("securityHotspots", securityHotspots)); + project.getTechnicalDebt().ifPresent(technicalDebt -> json.prop("technicalDebt", technicalDebt)); + project.getDevelopmentCost().ifPresent(developmentCost -> json.prop("developmentCost", developmentCost)); + project.getExternalSecurityReportExportedAt().ifPresent(exportedAt -> json.prop("externalSecurityReportExportedAt", exportedAt)); + json.endObject(); + }); + json.endArray(); + } + } + + private static void writeQualityGates(JsonWriter json, TelemetryData telemetryData) { + if (telemetryData.getQualityGates() != null) { + json.name("quality-gates"); + json.beginArray(); + telemetryData.getQualityGates().forEach(qualityGate -> { + json.beginObject(); + json.prop("uuid", qualityGate.uuid()); + json.prop("caycStatus", qualityGate.caycStatus()); + json.name("conditions"); + json.beginArray(); + qualityGate.conditions().forEach(condition -> { + json.beginObject(); + json.prop("metric", condition.getMetricKey()); + json.prop("comparison_operator", condition.getOperator().getDbValue()); + json.prop("error_value", condition.getErrorThreshold()); + json.endObject(); + }); + json.endArray(); + json.endObject(); + }); + json.endArray(); + } + } + + private static void writeQualityProfiles(JsonWriter json, TelemetryData telemetryData) { + if (telemetryData.getQualityProfiles() != null) { + json.name("quality-profiles"); + json.beginArray(); + telemetryData.getQualityProfiles().forEach(qualityProfile -> { + json.beginObject(); + json.prop("uuid", qualityProfile.uuid()); + json.prop("parentUuid", qualityProfile.parentUuid()); + json.prop(LANGUAGE_PROPERTY, qualityProfile.language()); + json.prop("default", qualityProfile.isDefault()); + json.prop("builtIn", qualityProfile.isBuiltIn()); + if (qualityProfile.builtInParent() != null) { + json.prop("builtInParent", qualityProfile.builtInParent()); + } + json.prop("rulesOverriddenCount", qualityProfile.rulesOverriddenCount()); + json.prop("rulesActivatedCount", qualityProfile.rulesActivatedCount()); + json.prop("rulesDeactivatedCount", qualityProfile.rulesDeactivatedCount()); + json.endObject(); + }); + json.endArray(); + } + } + private static void writeManagedInstanceInformation(JsonWriter json, TelemetryData.ManagedInstanceInformation provider) { + json.name(MANAGED_INSTANCE_PROPERTY); + json.beginObject(); + json.prop("isManaged", provider.isManaged()); + json.prop("provider", provider.isManaged() ? provider.provider() : null); + json.endObject(); + } + + private static void writeCloudUsage(JsonWriter json, TelemetryData.CloudUsage cloudUsage) { + json.name(CLOUD_USAGE_PROPERTY); + json.beginObject(); + json.prop("kubernetes", cloudUsage.kubernetes()); + json.prop("kubernetesVersion", cloudUsage.kubernetesVersion()); + json.prop("kubernetesPlatform", cloudUsage.kubernetesPlatform()); + json.prop("kubernetesProvider", cloudUsage.kubernetesProvider()); + json.prop("officialHelmChart", cloudUsage.officialHelmChart()); + json.prop("containerRuntime", cloudUsage.containerRuntime()); + json.prop("officialImage", cloudUsage.officialImage()); + json.endObject(); + } + + @NotNull + private static String toUtc(long date) { + return DateTimeFormatter.ofPattern(DATETIME_FORMAT) + .withZone(ZoneOffset.UTC) + .format(Instant.ofEpochMilli(date)); + } + +} diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDataLoader.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDataLoader.java new file mode 100644 index 00000000000..658c2deccee --- /dev/null +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDataLoader.java @@ -0,0 +1,28 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +public interface TelemetryDataLoader { + TelemetryData load(); + + String loadServerId(); + + void reset(); +} diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDataLoaderImpl.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDataLoaderImpl.java new file mode 100644 index 00000000000..183d7f3eca0 --- /dev/null +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/TelemetryDataLoaderImpl.java @@ -0,0 +1,535 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import javax.inject.Inject; +import org.sonar.api.config.Configuration; +import org.sonar.api.platform.Server; +import org.sonar.api.server.ServerSide; +import org.sonar.core.platform.PlatformEditionProvider; +import org.sonar.core.platform.PluginInfo; +import org.sonar.core.platform.PluginRepository; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.alm.setting.ALM; +import org.sonar.db.alm.setting.ProjectAlmKeyAndProject; +import org.sonar.db.component.AnalysisPropertyValuePerProject; +import org.sonar.db.component.BranchMeasuresDto; +import org.sonar.db.component.PrBranchAnalyzedLanguageCountByProjectDto; +import org.sonar.db.component.SnapshotDto; +import org.sonar.db.measure.ProjectLocDistributionDto; +import org.sonar.db.measure.ProjectMainBranchLiveMeasureDto; +import org.sonar.db.metric.MetricDto; +import org.sonar.db.newcodeperiod.NewCodePeriodDto; +import org.sonar.db.project.ProjectDto; +import org.sonar.db.property.PropertyDto; +import org.sonar.db.property.PropertyQuery; +import org.sonar.db.qualitygate.ProjectQgateAssociationDto; +import org.sonar.db.qualitygate.QualityGateConditionDto; +import org.sonar.db.qualitygate.QualityGateDto; +import org.sonar.server.management.ManagedInstanceService; +import org.sonar.server.platform.ContainerSupport; +import org.sonar.server.property.InternalProperties; +import org.sonar.server.qualitygate.Condition; +import org.sonar.server.qualitygate.QualityGateCaycChecker; +import org.sonar.server.qualitygate.QualityGateFinder; +import org.sonar.telemetry.deprecated.TelemetryData.Database; +import org.sonar.telemetry.deprecated.TelemetryData.NewCodeDefinition; + +import static java.util.Arrays.asList; +import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toMap; +import static org.apache.commons.lang3.StringUtils.startsWithIgnoreCase; +import static org.sonar.api.measures.CoreMetrics.BUGS_KEY; +import static org.sonar.api.measures.CoreMetrics.DEVELOPMENT_COST_KEY; +import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; +import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_KEY; +import static org.sonar.api.measures.CoreMetrics.TECHNICAL_DEBT_KEY; +import static org.sonar.api.measures.CoreMetrics.VULNERABILITIES_KEY; +import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI; +import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM; +import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY; +import static org.sonar.core.platform.EditionProvider.Edition.DATACENTER; +import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH; +import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY; +import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY; +import static org.sonar.server.qualitygate.Condition.Operator.fromDbValue; +import static org.sonar.telemetry.deprecated.TelemetryDaemon.I_PROP_MESSAGE_SEQUENCE; + +@ServerSide +public class TelemetryDataLoaderImpl implements TelemetryDataLoader { + private static final String UNDETECTED = "undetected"; + public static final String EXTERNAL_SECURITY_REPORT_EXPORTED_AT = "project.externalSecurityReportExportedAt"; + + private static final Map<String, String> LANGUAGES_BY_SECURITY_JSON_PROPERTY_MAP = Map.of( + "sonar.security.config.javasecurity", "java", + "sonar.security.config.phpsecurity", "php", + "sonar.security.config.pythonsecurity", "python", + "sonar.security.config.roslyn.sonaranalyzer.security.cs", "csharp"); + + private final Server server; + private final DbClient dbClient; + private final PluginRepository pluginRepository; + private final PlatformEditionProvider editionProvider; + private final Configuration configuration; + private final InternalProperties internalProperties; + private final ContainerSupport containerSupport; + private final QualityGateCaycChecker qualityGateCaycChecker; + private final QualityGateFinder qualityGateFinder; + private final ManagedInstanceService managedInstanceService; + private final CloudUsageDataProvider cloudUsageDataProvider; + private final QualityProfileDataProvider qualityProfileDataProvider; + private final Set<NewCodeDefinition> newCodeDefinitions = new HashSet<>(); + private final Map<String, NewCodeDefinition> ncdByProject = new HashMap<>(); + private final Map<String, NewCodeDefinition> ncdByBranch = new HashMap<>(); + private final Map<String, String> defaultQualityProfileByLanguage = new HashMap<>(); + private final Map<ProjectLanguageKey, String> qualityProfileByProjectAndLanguage = new HashMap<>(); + private NewCodeDefinition instanceNcd = NewCodeDefinition.getInstanceDefault(); + + @Inject + public TelemetryDataLoaderImpl(Server server, DbClient dbClient, PluginRepository pluginRepository, + PlatformEditionProvider editionProvider, InternalProperties internalProperties, Configuration configuration, + ContainerSupport containerSupport, QualityGateCaycChecker qualityGateCaycChecker, QualityGateFinder qualityGateFinder, + ManagedInstanceService managedInstanceService, CloudUsageDataProvider cloudUsageDataProvider, QualityProfileDataProvider qualityProfileDataProvider) { + this.server = server; + this.dbClient = dbClient; + this.pluginRepository = pluginRepository; + this.editionProvider = editionProvider; + this.internalProperties = internalProperties; + this.configuration = configuration; + this.containerSupport = containerSupport; + this.qualityGateCaycChecker = qualityGateCaycChecker; + this.qualityGateFinder = qualityGateFinder; + this.managedInstanceService = managedInstanceService; + this.cloudUsageDataProvider = cloudUsageDataProvider; + this.qualityProfileDataProvider = qualityProfileDataProvider; + } + + private static Database loadDatabaseMetadata(DbSession dbSession) { + try { + DatabaseMetaData metadata = dbSession.getConnection().getMetaData(); + return new Database(metadata.getDatabaseProductName(), metadata.getDatabaseProductVersion()); + } catch (SQLException e) { + throw new IllegalStateException("Fail to get DB metadata", e); + } + } + + @Override + public TelemetryData load() { + TelemetryData.Builder data = TelemetryData.builder(); + + data.setMessageSequenceNumber(retrieveCurrentMessageSequenceNumber() + 1); + data.setServerId(server.getId()); + data.setVersion(server.getVersion()); + data.setEdition(editionProvider.get().orElse(null)); + Function<PluginInfo, String> getVersion = plugin -> plugin.getVersion() == null ? "undefined" : plugin.getVersion().getName(); + Map<String, String> plugins = pluginRepository.getPluginInfos().stream().collect(toMap(PluginInfo::getKey, getVersion)); + data.setPlugins(plugins); + try (DbSession dbSession = dbClient.openSession(false)) { + var branchMeasuresDtos = dbClient.branchDao().selectBranchMeasuresWithCaycMetric(dbSession); + loadNewCodeDefinitions(dbSession, branchMeasuresDtos); + loadQualityProfiles(dbSession); + + data.setDatabase(loadDatabaseMetadata(dbSession)); + data.setNcdId(instanceNcd.hashCode()); + data.setNewCodeDefinitions(newCodeDefinitions); + + String defaultQualityGateUuid = qualityGateFinder.getDefault(dbSession).getUuid(); + String sonarWayQualityGateUuid = qualityGateFinder.getSonarWay(dbSession).getUuid(); + List<ProjectDto> projects = dbClient.projectDao().selectProjects(dbSession); + + data.setDefaultQualityGate(defaultQualityGateUuid); + data.setSonarWayQualityGate(sonarWayQualityGateUuid); + resolveUnanalyzedLanguageCode(data, dbSession); + resolveProjectStatistics(data, dbSession, defaultQualityGateUuid, projects); + resolveProjects(data, dbSession); + resolveBranches(data, branchMeasuresDtos); + resolveQualityGates(data, dbSession); + resolveUsers(data, dbSession); + } + + data.setQualityProfiles(qualityProfileDataProvider.retrieveQualityProfilesData()); + + setSecurityCustomConfigIfPresent(data); + + Optional<String> installationDateProperty = internalProperties.read(InternalProperties.INSTALLATION_DATE); + installationDateProperty.ifPresent(s -> data.setInstallationDate(Long.valueOf(s))); + Optional<String> installationVersionProperty = internalProperties.read(InternalProperties.INSTALLATION_VERSION); + + return data + .setInstallationVersion(installationVersionProperty.orElse(null)) + .setInContainer(containerSupport.isRunningInContainer()) + .setManagedInstanceInformation(buildManagedInstanceInformation()) + .setCloudUsage(buildCloudUsage()) + .build(); + } + + private void resolveBranches(TelemetryData.Builder data, List<BranchMeasuresDto> branchMeasuresDtos) { + var branches = branchMeasuresDtos.stream() + .map(dto -> { + var projectNcd = ncdByProject.getOrDefault(dto.getProjectUuid(), instanceNcd); + var ncdId = ncdByBranch.getOrDefault(dto.getBranchUuid(), projectNcd).hashCode(); + return new TelemetryData.Branch( + dto.getProjectUuid(), dto.getBranchUuid(), ncdId, + dto.getGreenQualityGateCount(), dto.getAnalysisCount(), dto.getExcludeFromPurge()); + }) + .toList(); + data.setBranches(branches); + } + + @Override + public void reset() { + this.newCodeDefinitions.clear(); + this.ncdByBranch.clear(); + this.ncdByProject.clear(); + this.instanceNcd = NewCodeDefinition.getInstanceDefault(); + this.defaultQualityProfileByLanguage.clear(); + this.qualityProfileByProjectAndLanguage.clear(); + } + + private void loadNewCodeDefinitions(DbSession dbSession, List<BranchMeasuresDto> branchMeasuresDtos) { + var branchUuidByKey = branchMeasuresDtos.stream() + .collect(Collectors.toMap(dto -> createBranchUniqueKey(dto.getProjectUuid(), dto.getBranchKey()), BranchMeasuresDto::getBranchUuid)); + List<NewCodePeriodDto> newCodePeriodDtos = dbClient.newCodePeriodDao().selectAll(dbSession); + NewCodeDefinition ncd; + boolean hasInstance = false; + for (var dto : newCodePeriodDtos) { + String projectUuid = dto.getProjectUuid(); + String branchUuid = dto.getBranchUuid(); + if (branchUuid == null && projectUuid == null) { + ncd = new NewCodeDefinition(dto.getType().name(), dto.getValue(), "instance"); + this.instanceNcd = ncd; + hasInstance = true; + } else if (projectUuid != null) { + var value = dto.getType() == REFERENCE_BRANCH ? branchUuidByKey.get(createBranchUniqueKey(projectUuid, dto.getValue())) : dto.getValue(); + if (branchUuid == null || isCommunityEdition()) { + ncd = new NewCodeDefinition(dto.getType().name(), value, "project"); + this.ncdByProject.put(projectUuid, ncd); + } else { + ncd = new NewCodeDefinition(dto.getType().name(), value, "branch"); + this.ncdByBranch.put(branchUuid, ncd); + } + } else { + throw new IllegalStateException(String.format("Error in loading telemetry data. New code definition for branch %s doesn't have a projectUuid", branchUuid)); + } + this.newCodeDefinitions.add(ncd); + } + if (!hasInstance) { + this.newCodeDefinitions.add(NewCodeDefinition.getInstanceDefault()); + } + } + + private void loadQualityProfiles(DbSession dbSession) { + dbClient.qualityProfileDao().selectAllDefaultProfiles(dbSession) + .forEach(defaultQualityProfile -> this.defaultQualityProfileByLanguage.put(defaultQualityProfile.getLanguage(), defaultQualityProfile.getKee())); + + dbClient.qualityProfileDao().selectAllProjectAssociations(dbSession) + .forEach(projectAssociation -> qualityProfileByProjectAndLanguage.put( + new ProjectLanguageKey(projectAssociation.projectUuid(), projectAssociation.language()), + projectAssociation.profileKey())); + } + + private boolean isCommunityEdition() { + var edition = editionProvider.get(); + return edition.isPresent() && edition.get() == COMMUNITY; + } + + private static String createBranchUniqueKey(String projectUuid, @Nullable String branchKey) { + return projectUuid + "-" + branchKey; + } + + private void resolveUnanalyzedLanguageCode(TelemetryData.Builder data, DbSession dbSession) { + long numberOfUnanalyzedCMeasures = dbClient.liveMeasureDao().countProjectsHavingMeasure(dbSession, UNANALYZED_C_KEY); + long numberOfUnanalyzedCppMeasures = dbClient.liveMeasureDao().countProjectsHavingMeasure(dbSession, UNANALYZED_CPP_KEY); + editionProvider.get() + .filter(edition -> edition.equals(COMMUNITY)) + .ifPresent(edition -> { + data.setHasUnanalyzedC(numberOfUnanalyzedCMeasures > 0); + data.setHasUnanalyzedCpp(numberOfUnanalyzedCppMeasures > 0); + }); + } + + private Long retrieveCurrentMessageSequenceNumber() { + return internalProperties.read(I_PROP_MESSAGE_SEQUENCE).map(Long::parseLong).orElse(0L); + } + + private void resolveProjectStatistics(TelemetryData.Builder data, DbSession dbSession, String defaultQualityGateUuid, List<ProjectDto> projects) { + Map<String, String> scmByProject = getAnalysisPropertyByProject(dbSession, SONAR_ANALYSIS_DETECTEDSCM); + Map<String, String> ciByProject = getAnalysisPropertyByProject(dbSession, SONAR_ANALYSIS_DETECTEDCI); + Map<String, ProjectAlmKeyAndProject> almAndUrlAndMonorepoByProject = getAlmAndUrlByProject(dbSession); + Map<String, PrBranchAnalyzedLanguageCountByProjectDto> prAndBranchCountByProject = dbClient.branchDao().countPrBranchAnalyzedLanguageByProjectUuid(dbSession) + .stream().collect(toMap(PrBranchAnalyzedLanguageCountByProjectDto::getProjectUuid, Function.identity())); + Map<String, String> qgatesByProject = getProjectQgatesMap(dbSession); + Map<String, Map<String, Number>> metricsByProject = getProjectMetricsByMetricKeys(dbSession, TECHNICAL_DEBT_KEY, DEVELOPMENT_COST_KEY, SECURITY_HOTSPOTS_KEY, + VULNERABILITIES_KEY, + BUGS_KEY); + Map<String, Long> securityReportExportedAtByProjectUuid = getSecurityReportExportedAtDateByProjectUuid(dbSession); + + List<TelemetryData.ProjectStatistics> projectStatistics = new ArrayList<>(); + for (ProjectDto project : projects) { + String projectUuid = project.getUuid(); + Map<String, Number> metrics = metricsByProject.getOrDefault(projectUuid, Collections.emptyMap()); + Optional<PrBranchAnalyzedLanguageCountByProjectDto> counts = ofNullable(prAndBranchCountByProject.get(projectUuid)); + + TelemetryData.ProjectStatistics stats = new TelemetryData.ProjectStatistics.Builder() + .setProjectUuid(projectUuid) + .setBranchCount(counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getBranch).orElse(0L)) + .setPRCount(counts.map(PrBranchAnalyzedLanguageCountByProjectDto::getPullRequest).orElse(0L)) + .setQG(qgatesByProject.getOrDefault(projectUuid, defaultQualityGateUuid)) + .setScm(Optional.ofNullable(scmByProject.get(projectUuid)).orElse(UNDETECTED)) + .setCi(Optional.ofNullable(ciByProject.get(projectUuid)).orElse(UNDETECTED)) + .setDevops(resolveDevopsPlatform(almAndUrlAndMonorepoByProject, projectUuid)) + .setBugs(metrics.getOrDefault("bugs", null)) + .setDevelopmentCost(metrics.getOrDefault("development_cost", null)) + .setVulnerabilities(metrics.getOrDefault("vulnerabilities", null)) + .setSecurityHotspots(metrics.getOrDefault("security_hotspots", null)) + .setTechnicalDebt(metrics.getOrDefault("sqale_index", null)) + .setNcdId(ncdByProject.getOrDefault(projectUuid, instanceNcd).hashCode()) + .setExternalSecurityReportExportedAt(securityReportExportedAtByProjectUuid.get(projectUuid)) + .setCreationMethod(project.getCreationMethod()) + .setMonorepo(resolveMonorepo(almAndUrlAndMonorepoByProject, projectUuid)) + .build(); + projectStatistics.add(stats); + } + data.setProjectStatistics(projectStatistics); + } + + private Map<String, Long> getSecurityReportExportedAtDateByProjectUuid(DbSession dbSession) { + PropertyQuery propertyQuery = PropertyQuery.builder().setKey(EXTERNAL_SECURITY_REPORT_EXPORTED_AT).build(); + List<PropertyDto> properties = dbClient.propertiesDao().selectByQuery(propertyQuery, dbSession); + return properties.stream() + .collect(toMap(PropertyDto::getEntityUuid, propertyDto -> Long.parseLong(propertyDto.getValue()))); + } + + private static String resolveDevopsPlatform(Map<String, ProjectAlmKeyAndProject> almAndUrlByProject, String projectUuid) { + if (almAndUrlByProject.containsKey(projectUuid)) { + ProjectAlmKeyAndProject projectAlmKeyAndProject = almAndUrlByProject.get(projectUuid); + return getAlmName(projectAlmKeyAndProject.getAlmId(), projectAlmKeyAndProject.getUrl()); + } + return UNDETECTED; + } + + private static Boolean resolveMonorepo(Map<String, ProjectAlmKeyAndProject> almAndUrlByProject, String projectUuid) { + return Optional.ofNullable(almAndUrlByProject.get(projectUuid)) + .map(ProjectAlmKeyAndProject::getMonorepo) + .orElse(false); + } + + private void resolveProjects(TelemetryData.Builder data, DbSession dbSession) { + Map<String, String> metricUuidMap = getNclocMetricUuidMap(dbSession); + String nclocUuid = metricUuidMap.get(NCLOC_KEY); + String nclocDistributionUuid = metricUuidMap.get(NCLOC_LANGUAGE_DISTRIBUTION_KEY); + List<ProjectLocDistributionDto> branchesWithLargestNcloc = dbClient.liveMeasureDao().selectLargestBranchesLocDistribution(dbSession, nclocUuid, nclocDistributionUuid); + List<String> branchUuids = branchesWithLargestNcloc.stream().map(ProjectLocDistributionDto::branchUuid).toList(); + Map<String, Long> latestSnapshotMap = dbClient.snapshotDao().selectLastAnalysesByRootComponentUuids(dbSession, branchUuids) + .stream() + .collect(toMap(SnapshotDto::getRootComponentUuid, SnapshotDto::getAnalysisDate)); + data.setProjects(buildProjectsList(branchesWithLargestNcloc, latestSnapshotMap)); + } + + private List<TelemetryData.Project> buildProjectsList(List<ProjectLocDistributionDto> branchesWithLargestNcloc, Map<String, Long> latestSnapshotMap) { + return branchesWithLargestNcloc.stream() + .flatMap(measure -> Arrays.stream(measure.locDistribution().split(";")) + .map(languageAndLoc -> languageAndLoc.split("=")) + .map(languageAndLoc -> new TelemetryData.Project( + measure.projectUuid(), + latestSnapshotMap.get(measure.branchUuid()), + languageAndLoc[0], + getQualityProfile(measure.projectUuid(), languageAndLoc[0]), + Long.parseLong(languageAndLoc[1])))) + .toList(); + } + + private String getQualityProfile(String projectUuid, String language) { + String qualityProfile = this.qualityProfileByProjectAndLanguage.get(new ProjectLanguageKey(projectUuid, language)); + if (qualityProfile != null) { + return qualityProfile; + } + return this.defaultQualityProfileByLanguage.get(language); + } + + private Map<String, String> getNclocMetricUuidMap(DbSession dbSession) { + return dbClient.metricDao().selectByKeys(dbSession, asList(NCLOC_KEY, NCLOC_LANGUAGE_DISTRIBUTION_KEY)) + .stream() + .collect(toMap(MetricDto::getKey, MetricDto::getUuid)); + } + + private void resolveQualityGates(TelemetryData.Builder data, DbSession dbSession) { + List<TelemetryData.QualityGate> qualityGates = new ArrayList<>(); + Collection<QualityGateDto> qualityGateDtos = dbClient.qualityGateDao().selectAll(dbSession); + Collection<QualityGateConditionDto> qualityGateConditions = dbClient.gateConditionDao().selectAll(dbSession); + Map<String, MetricDto> metricsByUuid = getMetricsByUuid(dbSession, qualityGateConditions); + + Map<String, List<Condition>> conditionsMap = mapQualityGateConditions(qualityGateConditions, metricsByUuid); + + for (QualityGateDto qualityGateDto : qualityGateDtos) { + String qualityGateUuid = qualityGateDto.getUuid(); + List<Condition> conditions = conditionsMap.getOrDefault(qualityGateUuid, Collections.emptyList()); + qualityGates.add( + new TelemetryData.QualityGate(qualityGateDto.getUuid(), qualityGateCaycChecker.checkCaycCompliant(dbSession, + qualityGateDto.getUuid()).toString(), conditions)); + } + + data.setQualityGates(qualityGates); + } + + private static Map<String, List<Condition>> mapQualityGateConditions(Collection<QualityGateConditionDto> qualityGateConditions, Map<String, MetricDto> metricsByUuid) { + Map<String, List<Condition>> conditionsMap = new HashMap<>(); + + for (QualityGateConditionDto condition : qualityGateConditions) { + String qualityGateUuid = condition.getQualityGateUuid(); + + MetricDto metricDto = metricsByUuid.get(condition.getMetricUuid()); + String metricKey = metricDto != null ? metricDto.getKey() : "Unknown Metric"; + + Condition telemetryCondition = new Condition( + metricKey, + fromDbValue(condition.getOperator()), + condition.getErrorThreshold()); + + conditionsMap + .computeIfAbsent(qualityGateUuid, k -> new ArrayList<>()) + .add(telemetryCondition); + } + + return conditionsMap; + } + + private Map<String, MetricDto> getMetricsByUuid(DbSession dbSession, Collection<QualityGateConditionDto> conditions) { + Set<String> metricUuids = conditions.stream().map(QualityGateConditionDto::getMetricUuid).collect(Collectors.toSet()); + return dbClient.metricDao().selectByUuids(dbSession, metricUuids).stream().filter(MetricDto::isEnabled).collect(Collectors.toMap(MetricDto::getUuid, Function.identity())); + } + + private void resolveUsers(TelemetryData.Builder data, DbSession dbSession) { + data.setUsers(dbClient.userDao().selectUsersForTelemetry(dbSession)); + } + + private void setSecurityCustomConfigIfPresent(TelemetryData.Builder data) { + editionProvider.get() + .filter(edition -> asList(ENTERPRISE, DATACENTER).contains(edition)) + .ifPresent(edition -> data.setCustomSecurityConfigs(getCustomerSecurityConfigurations())); + } + + private Map<String, String> getAnalysisPropertyByProject(DbSession dbSession, String analysisPropertyKey) { + return dbClient.analysisPropertiesDao() + .selectAnalysisPropertyValueInLastAnalysisPerProject(dbSession, analysisPropertyKey) + .stream() + .collect(toMap(AnalysisPropertyValuePerProject::getProjectUuid, AnalysisPropertyValuePerProject::getPropertyValue)); + } + + private Map<String, ProjectAlmKeyAndProject> getAlmAndUrlByProject(DbSession dbSession) { + List<ProjectAlmKeyAndProject> projectAlmKeyAndProjects = dbClient.projectAlmSettingDao().selectAlmTypeAndUrlByProject(dbSession); + return projectAlmKeyAndProjects.stream().collect(toMap(ProjectAlmKeyAndProject::getProjectUuid, Function.identity())); + } + + private static String getAlmName(String alm, String url) { + if (checkIfCloudAlm(alm, ALM.GITHUB.getId(), url, "https://api.github.com")) { + return "github_cloud"; + } + + if (checkIfCloudAlm(alm, ALM.GITLAB.getId(), url, "https://gitlab.com/api/v4")) { + return "gitlab_cloud"; + } + + if (checkIfCloudAlm(alm, ALM.AZURE_DEVOPS.getId(), url, "https://dev.azure.com")) { + return "azure_devops_cloud"; + } + + if (ALM.BITBUCKET_CLOUD.getId().equals(alm)) { + return alm; + } + + return alm + "_server"; + } + + private Map<String, String> getProjectQgatesMap(DbSession dbSession) { + return dbClient.projectQgateAssociationDao().selectAll(dbSession) + .stream() + .collect(toMap(ProjectQgateAssociationDto::getUuid, p -> Optional.ofNullable(p.getGateUuid()).orElse(""))); + } + + private Map<String, Map<String, Number>> getProjectMetricsByMetricKeys(DbSession dbSession, String... metricKeys) { + Map<String, String> metricNamesByUuid = dbClient.metricDao().selectByKeys(dbSession, asList(metricKeys)) + .stream() + .collect(toMap(MetricDto::getUuid, MetricDto::getKey)); + + // metrics can be empty for un-analyzed projects + if (metricNamesByUuid.isEmpty()) { + return Collections.emptyMap(); + } + + return dbClient.liveMeasureDao().selectForProjectMainBranchesByMetricUuids(dbSession, metricNamesByUuid.keySet()) + .stream() + .collect(groupingBy(ProjectMainBranchLiveMeasureDto::getProjectUuid, + toMap(lmDto -> metricNamesByUuid.get(lmDto.getMetricUuid()), + lmDto -> Optional.ofNullable(lmDto.getValue()).orElseGet(() -> Double.valueOf(lmDto.getTextValue())), + (oldValue, newValue) -> newValue, HashMap::new))); + } + + private static boolean checkIfCloudAlm(String almRaw, String alm, String url, String cloudUrl) { + return alm.equals(almRaw) && startsWithIgnoreCase(url, cloudUrl); + } + + @Override + public String loadServerId() { + return server.getId(); + } + + private Set<String> getCustomerSecurityConfigurations() { + return LANGUAGES_BY_SECURITY_JSON_PROPERTY_MAP.keySet().stream() + .filter(this::isPropertyPresentInConfiguration) + .map(LANGUAGES_BY_SECURITY_JSON_PROPERTY_MAP::get) + .collect(Collectors.toSet()); + } + + private boolean isPropertyPresentInConfiguration(String property) { + return configuration.get(property).isPresent(); + } + + private TelemetryData.ManagedInstanceInformation buildManagedInstanceInformation() { + String provider = managedInstanceService.isInstanceExternallyManaged() ? managedInstanceService.getProviderName() : null; + return new TelemetryData.ManagedInstanceInformation(managedInstanceService.isInstanceExternallyManaged(), provider); + } + + private TelemetryData.CloudUsage buildCloudUsage() { + return cloudUsageDataProvider.getCloudUsage(); + } + + private record ProjectLanguageKey(String projectKey, String language) { + } +} diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/package-info.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/package-info.java new file mode 100644 index 00000000000..6e334e48264 --- /dev/null +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/deprecated/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.telemetry.deprecated; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/CloudUsageDataProviderTest.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/CloudUsageDataProviderTest.java new file mode 100644 index 00000000000..ef7b9f0cde3 --- /dev/null +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/CloudUsageDataProviderTest.java @@ -0,0 +1,225 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Paths; +import javax.annotation.Nullable; +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.Before; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.server.platform.ContainerSupport; +import org.sonar.server.util.Paths2; +import org.sonarqube.ws.MediaTypes; + +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.telemetry.deprecated.CloudUsageDataProvider.DOCKER_RUNNING; +import static org.sonar.telemetry.deprecated.CloudUsageDataProvider.KUBERNETES_SERVICE_HOST; +import static org.sonar.telemetry.deprecated.CloudUsageDataProvider.KUBERNETES_SERVICE_PORT; +import static org.sonar.telemetry.deprecated.CloudUsageDataProvider.SONAR_HELM_CHART_VERSION; + +public class CloudUsageDataProviderTest { + + private final System2 system2 = mock(System2.class); + private final Paths2 paths2 = mock(Paths2.class); + private final OkHttpClient httpClient = mock(OkHttpClient.class); + private final ContainerSupport containerSupport = mock(ContainerSupport.class); + private final ProcessBuilder processBuilder = mock(ProcessBuilder.class); + private final CloudUsageDataProvider underTest = new CloudUsageDataProvider(containerSupport, system2, paths2, () -> processBuilder, + httpClient); + + @Before + public void setUp() throws Exception { + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn("localhost"); + when(system2.envVariable(KUBERNETES_SERVICE_PORT)).thenReturn("443"); + + mockHttpClientCall(200, "OK", ResponseBody.create(""" + { + "major": "1", + "minor": "25", + "gitVersion": "v1.25.3", + "gitCommit": "434bfd82814af038ad94d62ebe59b133fcb50506", + "gitTreeState": "clean", + "buildDate": "2022-11-02T03:24:50Z", + "goVersion": "go1.19.2", + "compiler": "gc", + "platform": "linux/arm64" + } + """, MediaType.parse(MediaTypes.JSON))); + } + + private void mockHttpClientCall(int code, String message, @Nullable ResponseBody body) throws IOException { + Call callMock = mock(Call.class); + when(callMock.execute()).thenReturn(new Response.Builder() + .request(new Request.Builder().url("http://any.test/").build()) + .protocol(Protocol.HTTP_1_1) + .code(code) + .message(message) + .body(body) + .build()); + when(httpClient.newCall(any())).thenReturn(callMock); + } + + @Test + public void containerRuntime_whenContainerSupportContextExists_shouldNotBeNull() { + when(containerSupport.getContainerContext()).thenReturn("docker"); + assertThat(underTest.getCloudUsage().containerRuntime()).isEqualTo("docker"); + } + + @Test + public void containerRuntime_whenContainerSupportContextMissing_shouldBeNull() { + when(containerSupport.getContainerContext()).thenReturn(null); + assertThat(underTest.getCloudUsage().containerRuntime()).isNull(); + } + + @Test + public void kubernetes_whenEnvVarExists_shouldReturnTrue() { + assertThat(underTest.getCloudUsage().kubernetes()).isTrue(); + } + + @Test + public void kubernetes_whenEnvVarDoesNotExist_shouldReturnFalse() { + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn(null); + assertThat(underTest.getCloudUsage().kubernetes()).isFalse(); + } + + @Test + public void kubernetesVersion_whenOnKubernetes_shouldReturnValue() { + assertThat(underTest.getCloudUsage().kubernetesVersion()).isEqualTo("1.25"); + } + + @Test + public void kubernetesVersion_whenNotOnKubernetes_shouldReturnNull() { + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn(null); + assertThat(underTest.getCloudUsage().kubernetesVersion()).isNull(); + } + + @Test + public void kubernetesVersion_whenApiCallFails_shouldReturnNull() throws IOException { + mockHttpClientCall(404, "not found", null); + assertThat(underTest.getCloudUsage().kubernetesVersion()).isNull(); + } + + @Test + public void kubernetesPlatform_whenOnKubernetes_shouldReturnValue() { + assertThat(underTest.getCloudUsage().kubernetesPlatform()).isEqualTo("linux/arm64"); + } + + @Test + public void kubernetesPlatform_whenNotOnKubernetes_shouldReturnNull() { + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn(null); + assertThat(underTest.getCloudUsage().kubernetesPlatform()).isNull(); + } + + @Test + public void kubernetesPlatform_whenApiCallFails_shouldReturnNull() throws IOException { + mockHttpClientCall(404, "not found", null); + assertThat(underTest.getCloudUsage().kubernetesPlatform()).isNull(); + } + + @Test + public void kubernetesProvider_shouldReturnValue() throws IOException { + Process processMock = mock(Process.class); + when(processMock.getInputStream()).thenReturn(new ByteArrayInputStream("some-provider".getBytes())); + when(processBuilder.command(any(String[].class))).thenReturn(processBuilder); + when(processBuilder.start()).thenReturn(processMock); + + assertThat(underTest.getCloudUsage().kubernetesProvider()).isEqualTo("some-provider"); + } + + @Test + public void kubernetesProvider_whenValueContainsNullChars_shouldReturnValueWithoutNullChars() throws IOException { + Process processMock = mock(Process.class); + when(processMock.getInputStream()).thenReturn(new ByteArrayInputStream("so\u0000me-prov\u0000ider".getBytes())); + when(processBuilder.command(any(String[].class))).thenReturn(processBuilder); + when(processBuilder.start()).thenReturn(processMock); + + assertThat(underTest.getCloudUsage().kubernetesProvider()).isEqualTo("some-provider"); + } + + @Test + public void officialHelmChart_whenEnvVarExists_shouldReturnValue() { + when(system2.envVariable(SONAR_HELM_CHART_VERSION)).thenReturn("10.1.0"); + assertThat(underTest.getCloudUsage().officialHelmChart()).isEqualTo("10.1.0"); + } + + @Test + public void officialHelmChart_whenEnvVarDoesNotExist_shouldReturnNull() { + when(system2.envVariable(SONAR_HELM_CHART_VERSION)).thenReturn(null); + assertThat(underTest.getCloudUsage().officialHelmChart()).isNull(); + } + + @Test + public void officialImage_whenEnvVarTrue_shouldReturnTrue() { + when(system2.envVariable(DOCKER_RUNNING)).thenReturn("True"); + assertThat(underTest.getCloudUsage().officialImage()).isTrue(); + } + + @Test + public void officialImage_whenEnvVarFalse_shouldReturnFalse() { + when(system2.envVariable(DOCKER_RUNNING)).thenReturn("False"); + assertThat(underTest.getCloudUsage().officialImage()).isFalse(); + } + + @Test + public void officialImage_whenEnvVarDoesNotExist_shouldReturnFalse() { + when(system2.envVariable(DOCKER_RUNNING)).thenReturn(null); + assertThat(underTest.getCloudUsage().officialImage()).isFalse(); + } + + @Test + public void initHttpClient_whenValidCertificate_shouldCreateClient() throws URISyntaxException { + when(paths2.get(anyString())).thenReturn(Paths.get(requireNonNull(getClass().getResource("dummy.crt")).toURI())); + + CloudUsageDataProvider provider = new CloudUsageDataProvider(containerSupport, system2, paths2); + assertThat(provider.getHttpClient()).isNotNull(); + } + + @Test + public void initHttpClient_whenNotOnKubernetes_shouldNotCreateClient() throws URISyntaxException { + when(paths2.get(anyString())).thenReturn(Paths.get(requireNonNull(getClass().getResource("dummy.crt")).toURI())); + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn(null); + + CloudUsageDataProvider provider = new CloudUsageDataProvider(containerSupport, system2, paths2); + assertThat(provider.getHttpClient()).isNull(); + } + + @Test + public void initHttpClient_whenCertificateNotFound_shouldFail() { + when(paths2.get(any())).thenReturn(Paths.get("dummy.crt")); + + CloudUsageDataProvider provider = new CloudUsageDataProvider(containerSupport, system2, paths2); + assertThat(provider.getHttpClient()).isNull(); + } +} diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/FakeServer.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/FakeServer.java new file mode 100644 index 00000000000..2aba4bd50b2 --- /dev/null +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/FakeServer.java @@ -0,0 +1,70 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import java.util.Date; +import org.sonar.api.platform.Server; + +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; + +class FakeServer extends Server { + private String id; + private String version; + + public FakeServer() { + this.id = randomAlphanumeric(20); + this.version = randomAlphanumeric(10); + } + + @Override + public String getId() { + return id; + } + + FakeServer setId(String id) { + this.id = id; + return this; + } + + @Override + public String getVersion() { + return this.version; + } + + public FakeServer setVersion(String version) { + this.version = version; + return this; + } + + @Override + public Date getStartedAt() { + return null; + } + + @Override + public String getContextPath() { + return null; + } + + @Override + public String getPublicRootUrl() { + return null; + } +} diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryClientCompressionTest.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryClientCompressionTest.java new file mode 100644 index 00000000000..2e06214a36f --- /dev/null +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryClientCompressionTest.java @@ -0,0 +1,62 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import java.io.IOException; +import java.util.Objects; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import okio.GzipSource; +import okio.Okio; +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.sonar.api.config.internal.MapSettings; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_URL; + +public class TelemetryClientCompressionTest { + + private final OkHttpClient okHttpClient = new OkHttpClient(); + private final MockWebServer telemetryServer = new MockWebServer(); + + @Test + public void payload_is_gzip_encoded() throws IOException, InterruptedException { + telemetryServer.enqueue(new MockResponse().setResponseCode(200)); + MapSettings settings = new MapSettings(); + settings.setProperty(SONAR_TELEMETRY_URL.getKey(), telemetryServer.url("/").toString()); + TelemetryClient underTest = new TelemetryClient(okHttpClient, settings.asConfig()); + underTest.start(); + underTest.upload("payload compressed with gzip"); + + RecordedRequest request = telemetryServer.takeRequest(); + + String contentType = Objects.requireNonNull(request.getHeader("content-type")); + assertThat(MediaType.parse(contentType)).isEqualTo(MediaType.parse("application/json; charset=utf-8")); + assertThat(request.getHeader("content-encoding")).isEqualTo("gzip"); + + GzipSource source = new GzipSource(request.getBody()); + String body = Okio.buffer(source).readUtf8(); + Assertions.assertThat(body).isEqualTo("payload compressed with gzip"); + } +} diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryClientTest.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryClientTest.java new file mode 100644 index 00000000000..c0adfc11629 --- /dev/null +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryClientTest.java @@ -0,0 +1,84 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import java.io.IOException; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okio.Buffer; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.sonar.api.config.internal.MapSettings; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_COMPRESSION; +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_URL; + +public class TelemetryClientTest { + + private static final String JSON = "{\"key\":\"value\"}"; + private static final String TELEMETRY_URL = "https://telemetry.com/url"; + + private OkHttpClient okHttpClient = mock(OkHttpClient.class, RETURNS_DEEP_STUBS); + private MapSettings settings = new MapSettings(); + + private TelemetryClient underTest = new TelemetryClient(okHttpClient, settings.asConfig()); + + @Test + public void upload() throws IOException { + ArgumentCaptor<Request> requestCaptor = ArgumentCaptor.forClass(Request.class); + settings.setProperty(SONAR_TELEMETRY_URL.getKey(), TELEMETRY_URL); + settings.setProperty(SONAR_TELEMETRY_COMPRESSION.getKey(), false); + underTest.start(); + + underTest.upload(JSON); + + verify(okHttpClient).newCall(requestCaptor.capture()); + Request request = requestCaptor.getValue(); + assertThat(request.method()).isEqualTo("POST"); + assertThat(request.body().contentType()).isEqualTo(MediaType.parse("application/json; charset=utf-8")); + Buffer body = new Buffer(); + request.body().writeTo(body); + assertThat(body.readUtf8()).isEqualTo(JSON); + assertThat(request.url()).hasToString(TELEMETRY_URL); + } + + @Test + public void opt_out() throws IOException { + ArgumentCaptor<Request> requestCaptor = ArgumentCaptor.forClass(Request.class); + settings.setProperty(SONAR_TELEMETRY_URL.getKey(), TELEMETRY_URL); + underTest.start(); + + underTest.optOut(JSON); + + verify(okHttpClient).newCall(requestCaptor.capture()); + Request request = requestCaptor.getValue(); + assertThat(request.method()).isEqualTo("DELETE"); + assertThat(request.body().contentType()).isEqualTo(MediaType.parse("application/json; charset=utf-8")); + Buffer body = new Buffer(); + request.body().writeTo(body); + assertThat(body.readUtf8()).isEqualTo(JSON); + assertThat(request.url()).hasToString(TELEMETRY_URL); + } +} diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryDaemonTest.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryDaemonTest.java new file mode 100644 index 00000000000..56a3ac858f8 --- /dev/null +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryDaemonTest.java @@ -0,0 +1,219 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import java.io.IOException; +import java.util.Collections; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.event.Level; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.api.impl.utils.TestSystem2; +import org.sonar.api.testfixtures.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.api.utils.text.JsonWriter; +import org.sonar.server.property.InternalProperties; +import org.sonar.server.property.MapInternalProperties; +import org.sonar.server.util.GlobalLockManager; +import org.sonar.server.util.GlobalLockManagerImpl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.after; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.sonar.api.utils.DateUtils.parseDate; +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_ENABLE; +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_FREQUENCY_IN_SECONDS; +import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_URL; + +public class TelemetryDaemonTest { + @Rule + public LogTester logger = new LogTester().setLevel(LoggerLevel.DEBUG); + + private static final long ONE_HOUR = 60 * 60 * 1_000L; + private static final long ONE_DAY = 24 * ONE_HOUR; + private static final TelemetryData SOME_TELEMETRY_DATA = TelemetryData.builder() + .setServerId("foo") + .setVersion("bar") + .setMessageSequenceNumber(1L) + .setPlugins(Collections.emptyMap()) + .setDatabase(new TelemetryData.Database("H2", "11")) + .build(); + + private final TelemetryClient client = mock(TelemetryClient.class); + private final InternalProperties internalProperties = spy(new MapInternalProperties()); + private final GlobalLockManager lockManager = mock(GlobalLockManagerImpl.class); + private final TestSystem2 system2 = new TestSystem2().setNow(System.currentTimeMillis()); + private final MapSettings settings = new MapSettings(); + + private final TelemetryDataLoader dataLoader = mock(TelemetryDataLoader.class); + private final TelemetryDataJsonWriter dataJsonWriter = mock(TelemetryDataJsonWriter.class); + private final TelemetryDaemon underTest = new TelemetryDaemon(dataLoader, dataJsonWriter, client, settings.asConfig(), internalProperties, lockManager, system2); + + @After + public void tearDown() { + underTest.stop(); + } + + @Test + public void send_data_via_client_at_startup_after_initial_delay() throws IOException { + initTelemetrySettingsToDefaultValues(); + when(lockManager.tryLock(any(), anyInt())).thenReturn(true); + settings.setProperty("sonar.telemetry.frequencyInSeconds", "1"); + when(dataLoader.load()).thenReturn(SOME_TELEMETRY_DATA); + mockDataJsonWriterDoingSomething(); + + underTest.start(); + + verify(client, timeout(4_000).atLeastOnce()).upload(anyString()); + verify(dataJsonWriter).writeTelemetryData(any(JsonWriter.class), same(SOME_TELEMETRY_DATA)); + } + + private void mockDataJsonWriterDoingSomething() { + doAnswer(t -> { + JsonWriter json = t.getArgument(0); + json.beginObject().prop("foo", "bar").endObject(); + return null; + }) + .when(dataJsonWriter) + .writeTelemetryData(any(), any()); + } + + @Test + public void check_if_should_send_data_periodically() throws IOException { + initTelemetrySettingsToDefaultValues(); + when(lockManager.tryLock(any(), anyInt())).thenReturn(true); + long now = system2.now(); + long twentyHoursAgo = now - (ONE_HOUR * 20L); + long oneDayAgo = now - ONE_DAY; + internalProperties.write("telemetry.lastPing", String.valueOf(twentyHoursAgo)); + settings.setProperty("sonar.telemetry.frequencyInSeconds", "1"); + when(dataLoader.load()).thenReturn(SOME_TELEMETRY_DATA); + mockDataJsonWriterDoingSomething(); + + underTest.start(); + + verify(dataJsonWriter, after(2_000).never()).writeTelemetryData(any(JsonWriter.class), same(SOME_TELEMETRY_DATA)); + verify(client, never()).upload(anyString()); + + internalProperties.write("telemetry.lastPing", String.valueOf(oneDayAgo)); + + verify(client, timeout(2_000)).upload(anyString()); + verify(dataJsonWriter).writeTelemetryData(any(JsonWriter.class), same(SOME_TELEMETRY_DATA)); + } + + @Test + public void do_not_send_data_if_last_ping_earlier_than_one_day_ago() throws IOException { + initTelemetrySettingsToDefaultValues(); + when(lockManager.tryLock(any(), anyInt())).thenReturn(true); + settings.setProperty("sonar.telemetry.frequencyInSeconds", "1"); + long now = system2.now(); + long twentyHoursAgo = now - (ONE_HOUR * 20L); + mockDataJsonWriterDoingSomething(); + + internalProperties.write("telemetry.lastPing", String.valueOf(twentyHoursAgo)); + underTest.start(); + + verify(client, after(2_000).never()).upload(anyString()); + } + + @Test + public void send_data_if_last_ping_is_over_one_day_ago() throws IOException { + initTelemetrySettingsToDefaultValues(); + when(lockManager.tryLock(any(), anyInt())).thenReturn(true); + settings.setProperty("sonar.telemetry.frequencyInSeconds", "1"); + long today = parseDate("2017-08-01").getTime(); + system2.setNow(today); + long oneDayAgo = today - ONE_DAY - ONE_HOUR; + internalProperties.write("telemetry.lastPing", String.valueOf(oneDayAgo)); + reset(internalProperties); + when(dataLoader.load()).thenReturn(SOME_TELEMETRY_DATA); + mockDataJsonWriterDoingSomething(); + + underTest.start(); + + verify(internalProperties, timeout(4_000)).write("telemetry.lastPing", String.valueOf(today)); + verify(client).upload(anyString()); + } + + @Test + public void opt_out_sent_once() throws IOException { + initTelemetrySettingsToDefaultValues(); + when(lockManager.tryLock(any(), anyInt())).thenReturn(true); + settings.setProperty("sonar.telemetry.frequencyInSeconds", "1"); + settings.setProperty("sonar.telemetry.enable", "false"); + mockDataJsonWriterDoingSomething(); + + underTest.start(); + underTest.start(); + + verify(client, after(2_000).never()).upload(anyString()); + verify(client, timeout(2_000).times(1)).optOut(anyString()); + assertThat(logger.logs(Level.INFO)).contains("Sharing of SonarQube statistics is disabled."); + } + + @Test + public void write_sequence_as_one_if_not_previously_present() { + initTelemetrySettingsToDefaultValues(); + when(lockManager.tryLock(any(), anyInt())).thenReturn(true); + settings.setProperty("sonar.telemetry.frequencyInSeconds", "1"); + mockDataJsonWriterDoingSomething(); + + underTest.start(); + + verify(internalProperties, timeout(4_000)).write("telemetry.messageSeq", "1"); + } + + @Test + public void write_sequence_correctly_incremented() { + initTelemetrySettingsToDefaultValues(); + when(lockManager.tryLock(any(), anyInt())).thenReturn(true); + settings.setProperty("sonar.telemetry.frequencyInSeconds", "1"); + internalProperties.write("telemetry.messageSeq", "10"); + mockDataJsonWriterDoingSomething(); + + underTest.start(); + + verify(internalProperties, timeout(4_000)).write("telemetry.messageSeq", "10"); + + // force another ping + internalProperties.write("telemetry.lastPing", String.valueOf(system2.now() - ONE_DAY)); + + verify(internalProperties, timeout(4_000)).write("telemetry.messageSeq", "11"); + } + + private void initTelemetrySettingsToDefaultValues() { + settings.setProperty(SONAR_TELEMETRY_ENABLE.getKey(), SONAR_TELEMETRY_ENABLE.getDefaultValue()); + settings.setProperty(SONAR_TELEMETRY_URL.getKey(), SONAR_TELEMETRY_URL.getDefaultValue()); + settings.setProperty(SONAR_TELEMETRY_FREQUENCY_IN_SECONDS.getKey(), SONAR_TELEMETRY_FREQUENCY_IN_SECONDS.getDefaultValue()); + } + +} diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryDataJsonWriterTest.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryDataJsonWriterTest.java new file mode 100644 index 00000000000..1041f2787b5 --- /dev/null +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/deprecated/TelemetryDataJsonWriterTest.java @@ -0,0 +1,792 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.deprecated; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.jetbrains.annotations.NotNull; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.text.JsonWriter; +import org.sonar.core.platform.EditionProvider; +import org.sonar.core.telemetry.TelemetryExtension; +import org.sonar.db.project.CreationMethod; +import org.sonar.db.user.UserTelemetryDto; +import org.sonar.server.qualitygate.Condition; +import org.sonar.server.util.DigestUtil; + +import static java.util.stream.Collectors.joining; +import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.NUMBER_OF_DAYS; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.PREVIOUS_VERSION; +import static org.sonar.server.qualitygate.Condition.Operator.fromDbValue; +import static org.sonar.test.JsonAssert.assertJson; + +@RunWith(DataProviderRunner.class) +public class TelemetryDataJsonWriterTest { + + private final Random random = new Random(); + + private final TelemetryExtension extension = mock(TelemetryExtension.class); + + private final System2 system2 = mock(System2.class); + + private final TelemetryDataJsonWriter underTest = new TelemetryDataJsonWriter(List.of(extension), system2); + + private static final int NCD_ID = 12345; + + private static final TelemetryData.NewCodeDefinition NCD_INSTANCE = new TelemetryData.NewCodeDefinition(PREVIOUS_VERSION.name(), "", "instance"); + private static final TelemetryData.NewCodeDefinition NCD_PROJECT = new TelemetryData.NewCodeDefinition(NUMBER_OF_DAYS.name(), "30", "project"); + + @Test + public void write_server_id_version_and_sequence() { + TelemetryData data = telemetryBuilder().build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "id": "%s", + "version": "%s", + "messageSequenceNumber": %s + } + """.formatted(data.getServerId(), data.getVersion(), data.getMessageSequenceNumber())); + } + + @Test + public void does_not_write_edition_if_null() { + TelemetryData data = telemetryBuilder().build(); + + String json = writeTelemetryData(data); + + assertThat(json).doesNotContain("edition"); + } + + @Test + @UseDataProvider("allEditions") + public void writes_edition_if_non_null(EditionProvider.Edition edition) { + TelemetryData data = telemetryBuilder() + .setEdition(edition) + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "edition": "%s" + } + """.formatted(edition.name().toLowerCase(Locale.ENGLISH))); + } + + @Test + public void writes_default_qg() { + TelemetryData data = telemetryBuilder() + .setDefaultQualityGate("default-qg") + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "defaultQualityGate": "%s" + } + """.formatted(data.getDefaultQualityGate())); + } + + @Test + public void writes_sonarWay_qg() { + TelemetryData data = telemetryBuilder() + .setSonarWayQualityGate("sonarWayUUID") + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "sonarway_quality_gate_uuid": "%s" + } + """.formatted(data.getSonarWayQualityGate())); + } + + @Test + public void writes_database() { + String name = randomAlphabetic(12); + String version = randomAlphabetic(10); + TelemetryData data = telemetryBuilder() + .setDatabase(new TelemetryData.Database(name, version)) + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "database": { + "name": "%s", + "version": "%s" + } + } + """.formatted(name, version)); + } + + @Test + public void writes_no_plugins() { + TelemetryData data = telemetryBuilder() + .setPlugins(Collections.emptyMap()) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo(""" + { + "plugins": [] + } + """); + } + + @Test + public void writes_all_plugins() { + Map<String, String> plugins = IntStream.range(0, 1 + random.nextInt(10)) + .boxed() + .collect(Collectors.toMap(i -> "P" + i, i1 -> "V" + i1)); + TelemetryData data = telemetryBuilder() + .setPlugins(plugins) + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "plugins": [%s] + } + """.formatted(plugins.entrySet().stream().map(e -> "{\"name\":\"" + e.getKey() + "\",\"version\":\"" + e.getValue() + "\"}").collect(joining(",")))); + } + + @Test + public void does_not_write_installation_date_if_null() { + TelemetryData data = telemetryBuilder() + .setInstallationDate(null) + .build(); + + String json = writeTelemetryData(data); + + assertThat(json).doesNotContain("installationDate"); + } + + @Test + public void write_installation_date_in_utc_format() { + TelemetryData data = telemetryBuilder() + .setInstallationDate(1_000L) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo(""" + { + "installationDate":"1970-01-01T00:00:01+0000" + } + """); + } + + @Test + public void does_not_write_installation_version_if_null() { + TelemetryData data = telemetryBuilder() + .setInstallationVersion(null) + .build(); + + String json = writeTelemetryData(data); + + assertThat(json).doesNotContain("installationVersion"); + } + + @Test + public void write_installation_version() { + String installationVersion = randomAlphabetic(5); + TelemetryData data = telemetryBuilder() + .setInstallationVersion(installationVersion) + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "installationVersion": "%s" + } + """.formatted(installationVersion)); + } + + @Test + @UseDataProvider("getFeatureFlagEnabledStates") + public void write_container_flag(boolean isIncontainer) { + TelemetryData data = telemetryBuilder() + .setInContainer(isIncontainer) + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "container": %s + } + """.formatted(isIncontainer)); + } + + @DataProvider + public static Object[][] getManagedInstanceData() { + return new Object[][] { + {true, "scim"}, + {true, "github"}, + {true, "gitlab"}, + {false, null}, + }; + } + + @Test + @UseDataProvider("getManagedInstanceData") + public void writeTelemetryData_encodesCorrectlyManagedInstanceInformation(boolean isManaged, String provider) { + TelemetryData data = telemetryBuilder() + .setManagedInstanceInformation(new TelemetryData.ManagedInstanceInformation(isManaged, provider)) + .build(); + + String json = writeTelemetryData(data); + + if (isManaged) { + assertJson(json).isSimilarTo(""" + { + "managedInstanceInformation": { + "isManaged": true, + "provider": "%s" + } + } + """.formatted(provider)); + } else { + assertJson(json).isSimilarTo(""" + { + "managedInstanceInformation": { + "isManaged": false + } + } + """); + } + } + + @Test + public void writeTelemetryData_shouldWriteCloudUsage() { + TelemetryData data = telemetryBuilder().build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "cloudUsage": { + "kubernetes": true, + "kubernetesVersion": "1.27", + "kubernetesPlatform": "linux/amd64", + "kubernetesProvider": "5.4.181-99.354.amzn2.x86_64", + "officialHelmChart": "10.1.0", + "officialImage": false, + "containerRuntime": "docker" + } + } + """); + } + + @Test + public void writes_has_unanalyzed_languages() { + TelemetryData data = telemetryBuilder() + .setHasUnanalyzedC(true) + .setHasUnanalyzedCpp(false) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo(""" + { + "hasUnanalyzedC": true, + "hasUnanalyzedCpp": false, + } + """); + } + + @Test + public void writes_security_custom_config() { + TelemetryData data = telemetryBuilder() + .setCustomSecurityConfigs(Set.of("php", "java")) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo(""" + { + "customSecurityConfig": ["php", "java"] + } + """); + } + + @Test + public void writes_local_timestamp() { + when(system2.now()).thenReturn(1000L); + + TelemetryData data = telemetryBuilder().build(); + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo(""" + { + "localTimestamp": "1970-01-01T00:00:01+0000" + } + """); + } + + @Test + public void writes_all_users_with_anonymous_md5_uuids() { + TelemetryData data = telemetryBuilder() + .setUsers(attachUsers()) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo(""" + { + "users": [ + { + "userUuid": "%s", + "status": "active", + "identityProvider": "gitlab", + "lastActivity": "1970-01-01T00:00:00+0000", + "lastSonarlintActivity": "1970-01-01T00:00:00+0000", + "managed": true + }, + { + "userUuid": "%s", + "status": "inactive", + "identityProvider": "gitlab", + "lastActivity": "1970-01-01T00:00:00+0000", + "lastSonarlintActivity": "1970-01-01T00:00:00+0000", + "managed": false + }, + { + "userUuid": "%s", + "status": "active", + "identityProvider": "gitlab", + "lastActivity": "1970-01-01T00:00:00+0000", + "lastSonarlintActivity": "1970-01-01T00:00:00+0000", + "managed": true + } + ] + } + """ + .formatted(DigestUtil.sha3_224Hex("uuid-0"), DigestUtil.sha3_224Hex("uuid-1"), DigestUtil.sha3_224Hex("uuid-2"))); + } + + @Test + public void writes_all_projects() { + TelemetryData data = telemetryBuilder() + .setProjects(attachProjects()) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo(""" + { + "projects": [ + { + "projectUuid": "uuid-0", + "lastAnalysis": "1970-01-01T00:00:00+0000", + "language": "lang-0", + "qualityProfile" : "qprofile-0", + "loc": 2 + }, + { + "projectUuid": "uuid-1", + "lastAnalysis": "1970-01-01T00:00:00+0000", + "language": "lang-1", + "qualityProfile" : "qprofile-1", + "loc": 4 + }, + { + "projectUuid": "uuid-2", + "lastAnalysis": "1970-01-01T00:00:00+0000", + "language": "lang-2", + "qualityProfile" : "qprofile-2", + "loc": 6 + } + ] + } + """); + } + + @Test + public void writeTelemetryData_whenAnalyzedLanguages_shouldwriteAllProjectsStats() { + TelemetryData data = telemetryBuilder() + .setProjectStatistics(attachProjectStatsWithMetrics()) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo(""" + { + "projects-general-stats": [ + { + "projectUuid": "uuid-0", + "branchCount": 2, + "pullRequestCount": 2, + "qualityGate": "qg-0", + "scm": "scm-0", + "ci": "ci-0", + "devopsPlatform": "devops-0", + "bugs": 2, + "vulnerabilities": 3, + "securityHotspots": 4, + "technicalDebt": 60, + "developmentCost": 30, + "ncdId": 12345, + "externalSecurityReportExportedAt": 1500000, + "project_creation_method": "LOCAL_API", + "monorepo": true + }, + { + "projectUuid": "uuid-1", + "branchCount": 4, + "pullRequestCount": 4, + "qualityGate": "qg-1", + "scm": "scm-1", + "ci": "ci-1", + "devopsPlatform": "devops-1", + "bugs": 4, + "vulnerabilities": 6, + "securityHotspots": 8, + "technicalDebt": 120, + "developmentCost": 60, + "ncdId": 12345, + "externalSecurityReportExportedAt": 1500001, + "project_creation_method": "LOCAL_API", + "monorepo": false + }, + { + "projectUuid": "uuid-2", + "branchCount": 6, + "pullRequestCount": 6, + "qualityGate": "qg-2", + "scm": "scm-2", + "ci": "ci-2", + "devopsPlatform": "devops-2", + "bugs": 6, + "vulnerabilities": 9, + "securityHotspots": 12, + "technicalDebt": 180, + "developmentCost": 90, + "ncdId": 12345, + "externalSecurityReportExportedAt": 1500002, + "project_creation_method": "LOCAL_API", + "monorepo": true + } + ] + } + """); + } + + @Test + public void writes_all_projects_stats_with_unanalyzed_languages() { + TelemetryData data = telemetryBuilder() + .setProjectStatistics(attachProjectStats()) + .build(); + + String json = writeTelemetryData(data); + assertThat(json).doesNotContain("hasUnanalyzedC", "hasUnanalyzedCpp"); + } + + @Test + public void writes_all_projects_stats_without_missing_metrics() { + TelemetryData data = telemetryBuilder() + .setProjectStatistics(attachProjectStats()) + .build(); + String json = writeTelemetryData(data); + assertThat(json).doesNotContain("bugs", "vulnerabilities", "securityHotspots", "technicalDebt", "developmentCost"); + } + + @Test + public void writes_all_quality_gates() { + TelemetryData data = telemetryBuilder() + .setQualityGates(attachQualityGates()) + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "quality-gates": [ + { + "uuid": "uuid-0", + "caycStatus": "non-compliant", + "conditions": [ + { + "metric": "new_coverage", + "comparison_operator": "LT", + "error_value": "80" + }, + { + "metric": "new_duplicated_lines_density", + "comparison_operator": "GT", + "error_value": "3" + } + ] + }, + { + "uuid": "uuid-1", + "caycStatus": "compliant", + "conditions": [ + { + "metric": "new_coverage", + "comparison_operator": "LT", + "error_value": "80" + }, + { + "metric": "new_duplicated_lines_density", + "comparison_operator": "GT", + "error_value": "3" + } + ] + }, + { + "uuid": "uuid-2", + "caycStatus": "over-compliant", + "conditions": [ + { + "metric": "new_coverage", + "comparison_operator": "LT", + "error_value": "80" + }, + { + "metric": "new_duplicated_lines_density", + "comparison_operator": "GT", + "error_value": "3" + } + ] + } + ] + } + """); + } + + @Test + public void writeTelemetryData_shouldWriteQualityProfiles() { + TelemetryData data = telemetryBuilder() + .setQualityProfiles(List.of( + new TelemetryData.QualityProfile("uuid-1", "parent-uuid-1", "js", true, false, true, 2, 3, 4), + new TelemetryData.QualityProfile("uuid-1", null, "js", false, true, null, null, null, null))) + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "quality-profiles": [ + { + "uuid": "uuid-1", + "parentUuid": "parent-uuid-1", + "language": "js", + "default": true, + "builtIn": false, + "builtInParent": true, + "rulesOverriddenCount": 2, + "rulesActivatedCount": 3, + "rulesDeactivatedCount": 4 + }, + { + "uuid": "uuid-1", + "language": "js", + "default": false, + "builtIn": true + } + ]} + """); + } + + @Test + public void writes_all_branches() { + TelemetryData data = telemetryBuilder() + .setBranches(attachBranches()) + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "branches": [ + { + "projectUuid": "projectUuid1", + "branchUuid": "branchUuid1", + "ncdId": 12345, + "greenQualityGateCount": 1, + "analysisCount": 2, + "excludeFromPurge": true + }, + { + "projectUuid": "projectUuid2", + "branchUuid": "branchUuid2", + "ncdId": 12345, + "greenQualityGateCount": 0, + "analysisCount": 2, + "excludeFromPurge": true + } + ] + } + """); + } + + @Test + public void writes_new_code_definitions() { + TelemetryData data = telemetryBuilder() + .setNewCodeDefinitions(attachNewCodeDefinitions()) + .build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "new-code-definitions": [ + { + "ncdId": %s, + "type": "%s", + "value": "%s", + "scope": "%s" + }, + { + "ncdId": %s, + "type": "%s", + "value": "%s", + "scope": "%s" + }, + ] + + } + """.formatted(NCD_INSTANCE.hashCode(), NCD_INSTANCE.type(), NCD_INSTANCE.value(), NCD_INSTANCE.scope(), NCD_PROJECT.hashCode(), + NCD_PROJECT.type(), NCD_PROJECT.value(), NCD_PROJECT.scope())); + } + + @Test + public void writes_instance_new_code_definition() { + TelemetryData data = telemetryBuilder().build(); + + String json = writeTelemetryData(data); + assertThat(json).contains("ncdId"); + + } + + private static TelemetryData.Builder telemetryBuilder() { + return TelemetryData.builder() + .setServerId("foo") + .setVersion("bar") + .setMessageSequenceNumber(1L) + .setPlugins(Collections.emptyMap()) + .setManagedInstanceInformation(new TelemetryData.ManagedInstanceInformation(false, null)) + .setCloudUsage(new TelemetryData.CloudUsage(true, "1.27", "linux/amd64", "5.4.181-99.354.amzn2.x86_64", "10.1.0", "docker", false)) + .setDatabase(new TelemetryData.Database("H2", "11")) + .setNcdId(NCD_ID); + } + + @NotNull + private static List<UserTelemetryDto> attachUsers() { + return IntStream.range(0, 3) + .mapToObj( + i -> new UserTelemetryDto().setUuid("uuid-" + i).setActive(i % 2 == 0).setLastConnectionDate(1L) + .setLastSonarlintConnectionDate(2L).setExternalIdentityProvider("gitlab").setScimUuid(i % 2 == 0 ? "scim-uuid-" + i : null)) + .toList(); + } + + private static List<TelemetryData.Project> attachProjects() { + return IntStream.range(0, 3).mapToObj(i -> new TelemetryData.Project("uuid-" + i, 1L, "lang-" + i, "qprofile-" + i, (i + 1L) * 2)).toList(); + } + + private static List<TelemetryData.ProjectStatistics> attachProjectStatsWithMetrics() { + return IntStream.range(0, 3).mapToObj(i -> getProjectStatisticsWithMetricBuilder(i).build()).toList(); + } + + private static List<TelemetryData.ProjectStatistics> attachProjectStats() { + return IntStream.range(0, 3).mapToObj(i -> getProjectStatisticsBuilder(i).build()).toList(); + } + + private static TelemetryData.ProjectStatistics.Builder getProjectStatisticsBuilder(int i) { + return new TelemetryData.ProjectStatistics.Builder() + .setProjectUuid("uuid-" + i) + .setBranchCount((i + 1L) * 2L) + .setPRCount((i + 1L) * 2L) + .setQG("qg-" + i).setCi("ci-" + i) + .setScm("scm-" + i) + .setDevops("devops-" + i) + .setNcdId(NCD_ID) + .setCreationMethod(CreationMethod.LOCAL_API) + .setMonorepo(false); + } + + private static TelemetryData.ProjectStatistics.Builder getProjectStatisticsWithMetricBuilder(int i) { + return getProjectStatisticsBuilder(i) + .setBugs((i + 1L) * 2) + .setVulnerabilities((i + 1L) * 3) + .setSecurityHotspots((i + 1L) * 4) + .setDevelopmentCost((i + 1L) * 30d) + .setTechnicalDebt((i + 1L) * 60d) + .setExternalSecurityReportExportedAt(1_500_000L + i) + .setCreationMethod(CreationMethod.LOCAL_API) + .setMonorepo(i % 2 == 0); + } + + private List<TelemetryData.QualityGate> attachQualityGates() { + List<Condition> qualityGateConditions = attachQualityGateConditions(); + return List.of(new TelemetryData.QualityGate("uuid-0", "non-compliant", qualityGateConditions), + new TelemetryData.QualityGate("uuid-1", "compliant", qualityGateConditions), + new TelemetryData.QualityGate("uuid-2", "over-compliant", qualityGateConditions)); + } + + private List<Condition> attachQualityGateConditions() { + return List.of(new Condition("new_coverage", fromDbValue("LT"), "80"), + new Condition("new_duplicated_lines_density", fromDbValue("GT"), "3")); + } + + private List<TelemetryData.Branch> attachBranches() { + return List.of(new TelemetryData.Branch("projectUuid1", "branchUuid1", NCD_ID, 1, 2, true), + new TelemetryData.Branch("projectUuid2", "branchUuid2", NCD_ID, 0, 2, true)); + } + + private List<TelemetryData.NewCodeDefinition> attachNewCodeDefinitions() { + return List.of(NCD_INSTANCE, NCD_PROJECT); + } + + @DataProvider + public static Object[][] allEditions() { + return Arrays.stream(EditionProvider.Edition.values()) + .map(t -> new Object[] {t}) + .toArray(Object[][]::new); + } + + private String writeTelemetryData(TelemetryData data) { + StringWriter jsonString = new StringWriter(); + try (JsonWriter json = JsonWriter.of(jsonString)) { + underTest.writeTelemetryData(json, data); + } + return jsonString.toString(); + } + + @DataProvider + public static Set<Boolean> getFeatureFlagEnabledStates() { + return Set.of(true, false); + } +} diff --git a/server/sonar-telemetry/src/test/resources/org/sonar/telemetry/deprecated/dummy.crt b/server/sonar-telemetry/src/test/resources/org/sonar/telemetry/deprecated/dummy.crt new file mode 100644 index 00000000000..10e8a4a760e --- /dev/null +++ b/server/sonar-telemetry/src/test/resources/org/sonar/telemetry/deprecated/dummy.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICQjCCAaugAwIBAgIBADANBgkqhkiG9w0BAQ0FADA9MQswCQYDVQQGEwJ1czEO +MAwGA1UECAwFZHVtbXkxDjAMBgNVBAoMBWR1bW15MQ4wDAYDVQQDDAVkdW1teTAg +Fw0yMzA2MDkxMDMxMzRaGA8yMjk3MDMyNDEwMzEzNFowPTELMAkGA1UEBhMCdXMx +DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEOMAwGA1UEAwwFZHVtbXkw +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAPL0Byqouz9UNBFRLqRRuNdGniwh +LzheMFKsdQIasTddfbsne6IuqMIBRyNr/icPrxXZEx/LY7mlKpBCYM/yty5ngEon +0QTTw/GXj3A7eDcpYD/0pVRKFcKNFIp58IKV09to2h4ttQUdjMqLS2yjc0ADugmy +ctlTR90Yna31Gi/nAgMBAAGjUDBOMB0GA1UdDgQWBBSMaHVg1zegjAH8CEZdN87I +FtN/6jAfBgNVHSMEGDAWgBSMaHVg1zegjAH8CEZdN87IFtN/6jAMBgNVHRMEBTAD +AQH/MA0GCSqGSIb3DQEBDQUAA4GBAFRViPwyPMBY6auUmaywjeLqtVPfn58MNssN +TZEh4ft3d2Z531m5thtSiZhnKFU/f1xRecUXK3jew8/RAKVSsTH7A4NYfhu5Bs/K +JfFWv7NYwL5ntnaBQZQ5uSYwPwiTYZzFrTgEDDOkXpsf7g5A16hS/L11A1lwx9b4 +WWCrjmv4 +-----END CERTIFICATE----- |