source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2022 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.server.telemetry;
  21
  22 import java.sql.DatabaseMetaData;
  23 import java.sql.SQLException;
  24 import java.util.List;
  25 import java.util.Optional;
  26 import java.util.stream.Collectors;
  27 import java.util.stream.IntStream;
  28 import org.junit.Rule;
  29 import org.junit.Test;
  30 import org.sonar.api.config.Configuration;
  31 import org.sonar.api.impl.utils.TestSystem2;
  32 import org.sonar.core.platform.PlatformEditionProvider;
  33 import org.sonar.core.platform.PluginInfo;
  34 import org.sonar.core.platform.PluginRepository;
  35 import org.sonar.db.DbSession;
  36 import org.sonar.db.DbTester;
  37 import org.sonar.db.alm.setting.AlmSettingDto;
  38 import org.sonar.db.component.AnalysisPropertyDto;
  39 import org.sonar.db.component.ComponentDto;
  40 import org.sonar.db.component.SnapshotDto;
  41 import org.sonar.db.metric.MetricDto;
  42 import org.sonar.db.user.UserDto;
  43 import org.sonar.db.user.UserTelemetryDto;
  44 import org.sonar.server.platform.DockerSupport;
  45 import org.sonar.server.property.InternalProperties;
  46 import org.sonar.server.property.MapInternalProperties;
  47 import org.sonar.updatecenter.common.Version;
  48
  49 import static java.util.Arrays.asList;
  50 import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
  51 import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
  52 import static org.assertj.core.api.Assertions.assertThat;
  53 import static org.assertj.core.api.Assertions.entry;
  54 import static org.assertj.core.groups.Tuple.tuple;
  55 import static org.mockito.Mockito.mock;
  56 import static org.mockito.Mockito.spy;
  57 import static org.mockito.Mockito.when;
  58 import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY;
  59 import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
  60 import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
  61 import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY;
  62 import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI;
  63 import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM;
  64 import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY;
  65 import static org.sonar.core.platform.EditionProvider.Edition.DEVELOPER;
  66 import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE;
  67 import static org.sonar.db.component.BranchType.BRANCH;
  68 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY;
  69 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY;
  70
  71 public class TelemetryDataLoaderImplTest {
  72   private final static Long NOW = 100_000_000L;
  73   private final TestSystem2 system2 = new TestSystem2().setNow(NOW);
  74
  75   @Rule
  76   public DbTester db = DbTester.create(system2);
  77
  78   private final FakeServer server = new FakeServer();
  79   private final PluginRepository pluginRepository = mock(PluginRepository.class);
  80   private final Configuration configuration = mock(Configuration.class);
  81   private final PlatformEditionProvider editionProvider = mock(PlatformEditionProvider.class);
  82   private final DockerSupport dockerSupport = mock(DockerSupport.class);
  83   private final InternalProperties internalProperties = spy(new MapInternalProperties());
  84   private final LicenseReader licenseReader = mock(LicenseReader.class);
  85
  86   private final TelemetryDataLoader communityUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
  87       internalProperties, configuration, dockerSupport, null);
  88   private final TelemetryDataLoader commercialUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
  89       internalProperties, configuration, dockerSupport, licenseReader);
  90
  91   @Test
  92   public void send_telemetry_data() {
  93     String serverId = "AU-TpxcB-iU5OvuD2FL7";
  94     String version = "7.5.4";
  95     Long analysisDate = 1L;
  96     Long lastConnectionDate = 5L;
  97
  98     server.setId(serverId);
  99     server.setVersion(version);
 100     List<PluginInfo> plugins = asList(newPlugin("java", "4.12.0.11033"), newPlugin("scmgit", "1.2"), new PluginInfo("other"));
 101     when(pluginRepository.getPluginInfos()).thenReturn(plugins);
 102     when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER));
 103
 104     int activeUserCount = 3;
 105     List<UserDto> activeUsers = IntStream.range(0, activeUserCount).mapToObj(i -> db.users().insertUser(
 106       u -> u.setExternalIdentityProvider("provider" + i).setLastSonarlintConnectionDate(i * 2L)))
 107       .collect(Collectors.toList());
 108
 109     // update last connection
 110     activeUsers.forEach(u -> db.users().updateLastConnectionDate(u, 5L));
 111
 112     UserDto inactiveUser = db.users().insertUser(u -> u.setActive(false).setExternalIdentityProvider("provider0"));
 113
 114     MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY));
 115     MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY));
 116     MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY));
 117     MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY));
 118
 119     ComponentDto project1 = db.components().insertPrivateProject();
 120     db.measures().insertLiveMeasure(project1, lines, m -> m.setValue(110d));
 121     db.measures().insertLiveMeasure(project1, ncloc, m -> m.setValue(110d));
 122     db.measures().insertLiveMeasure(project1, coverage, m -> m.setValue(80d));
 123     db.measures().insertLiveMeasure(project1, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10"));
 124
 125     ComponentDto project2 = db.components().insertPrivateProject();
 126     db.measures().insertLiveMeasure(project2, lines, m -> m.setValue(200d));
 127     db.measures().insertLiveMeasure(project2, ncloc, m -> m.setValue(200d));
 128     db.measures().insertLiveMeasure(project2, coverage, m -> m.setValue(80d));
 129     db.measures().insertLiveMeasure(project2, nclocDistrib, m -> m.setValue(null).setData("java=180;js=20"));
 130
 131     SnapshotDto project1Analysis = db.components().insertSnapshot(project1, t -> t.setLast(true).setBuildDate(analysisDate));
 132     SnapshotDto project2Analysis = db.components().insertSnapshot(project2, t -> t.setLast(true).setBuildDate(analysisDate));
 133     db.measures().insertMeasure(project1, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10"));
 134     db.measures().insertMeasure(project2, project2Analysis, nclocDistrib, m -> m.setData("java=180;js=20"));
 135
 136     insertAnalysisProperty(project1Analysis, "prop-uuid-1", SONAR_ANALYSIS_DETECTEDCI, "ci-1");
 137     insertAnalysisProperty(project2Analysis, "prop-uuid-2", SONAR_ANALYSIS_DETECTEDCI, "ci-2");
 138     insertAnalysisProperty(project1Analysis, "prop-uuid-3", SONAR_ANALYSIS_DETECTEDSCM, "scm-1");
 139     insertAnalysisProperty(project2Analysis, "prop-uuid-4", SONAR_ANALYSIS_DETECTEDSCM, "scm-2");
 140
 141     // alm
 142     db.almSettings().insertAzureAlmSetting();
 143     db.almSettings().insertGitHubAlmSetting();
 144     AlmSettingDto almSettingDto = db.almSettings().insertAzureAlmSetting(a -> a.setUrl("https://dev.azure.com"));
 145     AlmSettingDto gitHubAlmSetting = db.almSettings().insertGitHubAlmSetting(a -> a.setUrl("https://api.github.com"));
 146     db.almSettings().insertAzureProjectAlmSetting(almSettingDto, db.components().getProjectDto(project1));
 147     db.almSettings().insertGitlabProjectAlmSetting(gitHubAlmSetting, db.components().getProjectDto(project2));
 148
 149     TelemetryData data = communityUnderTest.load();
 150     assertThat(data.getServerId()).isEqualTo(serverId);
 151     assertThat(data.getVersion()).isEqualTo(version);
 152     assertThat(data.getEdition()).contains(DEVELOPER);
 153     assertDatabaseMetadata(data.getDatabase());
 154     assertThat(data.getPlugins()).containsOnly(
 155       entry("java", "4.12.0.11033"), entry("scmgit", "1.2"), entry("other", "undefined"));
 156     assertThat(data.isInDocker()).isFalse();
 157
 158     assertThat(data.getUserTelemetries())
 159       .extracting(UserTelemetryDto::getUuid, UserTelemetryDto::getLastConnectionDate, UserTelemetryDto::getLastSonarlintConnectionDate, UserTelemetryDto::isActive)
 160       .containsExactlyInAnyOrder(
 161         tuple(activeUsers.get(0).getUuid(), lastConnectionDate, activeUsers.get(0).getLastSonarlintConnectionDate(), true),
 162         tuple(activeUsers.get(1).getUuid(), lastConnectionDate, activeUsers.get(1).getLastSonarlintConnectionDate(), true),
 163         tuple(activeUsers.get(2).getUuid(), lastConnectionDate, activeUsers.get(2).getLastSonarlintConnectionDate(), true),
 164         tuple(inactiveUser.getUuid(), null, inactiveUser.getLastSonarlintConnectionDate(), false));
 165     assertThat(data.getProjects())
 166       .extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc, TelemetryData.Project::getLastAnalysis)
 167       .containsExactlyInAnyOrder(
 168         tuple(project1.uuid(), "java", 70L, analysisDate),
 169         tuple(project1.uuid(), "js", 30L, analysisDate),
 170         tuple(project1.uuid(), "kotlin", 10L, analysisDate),
 171         tuple(project2.uuid(), "java", 180L, analysisDate),
 172         tuple(project2.uuid(), "js", 20L, analysisDate));
 173     assertThat(data.getProjectStatistics())
 174       .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount,
 175         TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi, TelemetryData.ProjectStatistics::getDevopsPlatform)
 176       .containsExactlyInAnyOrder(
 177         tuple(1L, 0L, "scm-1", "ci-1", "azure_devops_cloud"),
 178         tuple(1L, 0L, "scm-2", "ci-2", "github_cloud"));
 179   }
 180
 181   private void assertDatabaseMetadata(TelemetryData.Database database) {
 182     try (DbSession dbSession = db.getDbClient().openSession(false)) {
 183       DatabaseMetaData metadata = dbSession.getConnection().getMetaData();
 184       assertThat(database.getName()).isEqualTo("H2");
 185       assertThat(database.getVersion()).isEqualTo(metadata.getDatabaseProductVersion());
 186     } catch (SQLException e) {
 187       throw new RuntimeException(e);
 188     }
 189   }
 190
 191   @Test
 192   public void take_largest_branch_snapshot_project_data() {
 193     server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4");
 194
 195     MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY));
 196     MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY));
 197     MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY));
 198     MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY));
 199
 200     ComponentDto project = db.components().insertPublicProject();
 201     db.measures().insertLiveMeasure(project, lines, m -> m.setValue(110d));
 202     db.measures().insertLiveMeasure(project, ncloc, m -> m.setValue(110d));
 203     db.measures().insertLiveMeasure(project, coverage, m -> m.setValue(80d));
 204     db.measures().insertLiveMeasure(project, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10"));
 205
 206     ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setBranchType(BRANCH));
 207     db.measures().insertLiveMeasure(branch, lines, m -> m.setValue(180d));
 208     db.measures().insertLiveMeasure(branch, ncloc, m -> m.setValue(180d));
 209     db.measures().insertLiveMeasure(branch, coverage, m -> m.setValue(80d));
 210     db.measures().insertLiveMeasure(branch, nclocDistrib, m -> m.setValue(null).setData("java=100;js=50;kotlin=30"));
 211
 212     SnapshotDto project1Analysis = db.components().insertSnapshot(project, t -> t.setLast(true));
 213     SnapshotDto project2Analysis = db.components().insertSnapshot(branch, t -> t.setLast(true));
 214     db.measures().insertMeasure(project, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10"));
 215     db.measures().insertMeasure(branch, project2Analysis, nclocDistrib, m -> m.setData("java=100;js=50;kotlin=30"));
 216
 217     TelemetryData data = communityUnderTest.load();
 218
 219     assertThat(data.getProjects()).extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc)
 220       .containsExactlyInAnyOrder(
 221         tuple(project.uuid(), "java", 100L),
 222         tuple(project.uuid(), "js", 50L),
 223         tuple(project.uuid(), "kotlin", 30L));
 224     assertThat(data.getProjectStatistics())
 225       .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount,
 226         TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
 227       .containsExactlyInAnyOrder(
 228         tuple(2L, 0L, "undetected", "undetected"));
 229   }
 230
 231   @Test
 232   public void data_contains_no_license_type_on_community_edition() {
 233     TelemetryData data = communityUnderTest.load();
 234
 235     assertThat(data.getLicenseType()).isEmpty();
 236   }
 237
 238   @Test
 239   public void data_contains_no_license_type_on_commercial_edition_if_no_license() {
 240     when(licenseReader.read()).thenReturn(Optional.empty());
 241
 242     TelemetryData data = commercialUnderTest.load();
 243
 244     assertThat(data.getLicenseType()).isEmpty();
 245   }
 246
 247   @Test
 248   public void data_contains_weekly_count_sonarlint_users() {
 249     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 100_000L));
 250     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW));
 251     // these don't count
 252     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 1_000_000_000L));
 253     db.users().insertUser();
 254
 255     TelemetryData data = communityUnderTest.load();
 256     assertThat(data.getUserTelemetries())
 257       .hasSize(4);
 258   }
 259
 260   @Test
 261   public void data_has_license_type_on_commercial_edition_if_no_license() {
 262     String licenseType = randomAlphabetic(12);
 263     LicenseReader.License license = mock(LicenseReader.License.class);
 264     when(license.getType()).thenReturn(licenseType);
 265     when(licenseReader.read()).thenReturn(Optional.of(license));
 266
 267     TelemetryData data = commercialUnderTest.load();
 268
 269     assertThat(data.getLicenseType()).contains(licenseType);
 270   }
 271
 272   @Test
 273   public void send_server_id_and_version() {
 274     String id = randomAlphanumeric(40);
 275     String version = randomAlphanumeric(10);
 276     server.setId(id);
 277     server.setVersion(version);
 278
 279     TelemetryData data = communityUnderTest.load();
 280     assertThat(data.getServerId()).isEqualTo(id);
 281     assertThat(data.getVersion()).isEqualTo(version);
 282
 283     data = commercialUnderTest.load();
 284     assertThat(data.getServerId()).isEqualTo(id);
 285     assertThat(data.getVersion()).isEqualTo(version);
 286   }
 287
 288   @Test
 289   public void send_server_installation_date_and_installation_version() {
 290     String installationVersion = "7.9.BEST.LTS.EVER";
 291     Long installationDate = 1546300800000L; // 2019/01/01
 292     internalProperties.write(InternalProperties.INSTALLATION_DATE, String.valueOf(installationDate));
 293     internalProperties.write(InternalProperties.INSTALLATION_VERSION, installationVersion);
 294
 295     TelemetryData data = communityUnderTest.load();
 296
 297     assertThat(data.getInstallationDate()).isEqualTo(installationDate);
 298     assertThat(data.getInstallationVersion()).isEqualTo(installationVersion);
 299   }
 300
 301   @Test
 302   public void do_not_send_server_installation_details_if_missing_property() {
 303     TelemetryData data = communityUnderTest.load();
 304     assertThat(data.getInstallationDate()).isNull();
 305     assertThat(data.getInstallationVersion()).isNull();
 306
 307     data = commercialUnderTest.load();
 308     assertThat(data.getInstallationDate()).isNull();
 309     assertThat(data.getInstallationVersion()).isNull();
 310   }
 311
 312   @Test
 313   public void send_unanalyzed_languages_flags_when_edition_is_community() {
 314     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
 315     MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY));
 316     MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY));
 317     ComponentDto project1 = db.components().insertPublicProject();
 318     ComponentDto project2 = db.components().insertPublicProject();
 319     db.measures().insertLiveMeasure(project1, unanalyzedC);
 320     db.measures().insertLiveMeasure(project2, unanalyzedC);
 321     db.measures().insertLiveMeasure(project2, unanalyzedCpp);
 322
 323     TelemetryData data = communityUnderTest.load();
 324
 325     assertThat(data.hasUnanalyzedC().get()).isTrue();
 326     assertThat(data.hasUnanalyzedCpp().get()).isTrue();
 327   }
 328
 329   @Test
 330   public void do_not_send_unanalyzed_languages_flags_when_edition_is_not_community() {
 331     when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER));
 332     MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY));
 333     MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY));
 334     ComponentDto project1 = db.components().insertPublicProject();
 335     ComponentDto project2 = db.components().insertPublicProject();
 336     db.measures().insertLiveMeasure(project1, unanalyzedC);
 337     db.measures().insertLiveMeasure(project2, unanalyzedCpp);
 338
 339     TelemetryData data = communityUnderTest.load();
 340
 341     assertThat(data.hasUnanalyzedC()).isEmpty();
 342     assertThat(data.hasUnanalyzedCpp()).isEmpty();
 343   }
 344
 345   @Test
 346   public void unanalyzed_languages_flags_are_set_to_false_when_no_unanalyzed_languages_and_edition_is_community() {
 347     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
 348
 349     TelemetryData data = communityUnderTest.load();
 350
 351     assertThat(data.hasUnanalyzedC().get()).isFalse();
 352     assertThat(data.hasUnanalyzedCpp().get()).isFalse();
 353   }
 354
 355   @Test
 356   public void populate_security_custom_config_for_languages_on_enterprise() {
 357     when(editionProvider.get()).thenReturn(Optional.of(ENTERPRISE));
 358
 359     when(configuration.get("sonar.security.config.javasecurity")).thenReturn(Optional.of("{}"));
 360     when(configuration.get("sonar.security.config.phpsecurity")).thenReturn(Optional.of("{}"));
 361     when(configuration.get("sonar.security.config.pythonsecurity")).thenReturn(Optional.of("{}"));
 362     when(configuration.get("sonar.security.config.roslyn.sonaranalyzer.security.cs")).thenReturn(Optional.of("{}"));
 363
 364     TelemetryData data = commercialUnderTest.load();
 365
 366     assertThat(data.getCustomSecurityConfigs())
 367       .containsExactlyInAnyOrder("java", "php", "python", "csharp");
 368   }
 369
 370   @Test
 371   public void skip_security_custom_config_on_community() {
 372     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
 373
 374     TelemetryData data = communityUnderTest.load();
 375
 376     assertThat(data.getCustomSecurityConfigs()).isEmpty();
 377   }
 378
 379   @Test
 380   public void undetected_alm_ci_slm_data() {
 381     server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4");
 382     db.components().insertPublicProject();
 383     TelemetryData data = communityUnderTest.load();
 384     assertThat(data.getProjectStatistics())
 385       .extracting(TelemetryData.ProjectStatistics::getDevopsPlatform, TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
 386       .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected"));
 387   }
 388
 389   private PluginInfo newPlugin(String key, String version) {
 390     return new PluginInfo(key)
 391       .setVersion(Version.create(version));
 392   }
 393
 394   private void insertAnalysisProperty(SnapshotDto snapshotDto, String uuid, String key, String value) {
 395     db.getDbClient().analysisPropertiesDao().insert(db.getSession(), new AnalysisPropertyDto()
 396       .setUuid(uuid)
 397       .setAnalysisUuid(snapshotDto.getUuid())
 398       .setKey(key)
 399       .setValue(value)
 400       .setCreatedAt(1L));
 401   }
 402
 403 }