source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2022 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.server.telemetry;
  21
  22 import java.sql.DatabaseMetaData;
  23 import java.sql.SQLException;
  24 import java.util.List;
  25 import java.util.Optional;
  26 import java.util.stream.Collectors;
  27 import java.util.stream.IntStream;
  28 import org.junit.Rule;
  29 import org.junit.Test;
  30 import org.sonar.api.config.Configuration;
  31 import org.sonar.api.impl.utils.TestSystem2;
  32 import org.sonar.core.platform.PlatformEditionProvider;
  33 import org.sonar.core.platform.PluginInfo;
  34 import org.sonar.core.platform.PluginRepository;
  35 import org.sonar.db.DbSession;
  36 import org.sonar.db.DbTester;
  37 import org.sonar.db.alm.setting.AlmSettingDto;
  38 import org.sonar.db.component.AnalysisPropertyDto;
  39 import org.sonar.db.component.ComponentDto;
  40 import org.sonar.db.component.SnapshotDto;
  41 import org.sonar.db.metric.MetricDto;
  42 import org.sonar.db.user.UserDto;
  43 import org.sonar.db.user.UserTelemetryDto;
  44 import org.sonar.server.es.EsTester;
  45 import org.sonar.server.measure.index.ProjectMeasuresIndex;
  46 import org.sonar.server.measure.index.ProjectMeasuresIndexer;
  47 import org.sonar.server.platform.DockerSupport;
  48 import org.sonar.server.property.InternalProperties;
  49 import org.sonar.server.property.MapInternalProperties;
  50 import org.sonar.updatecenter.common.Version;
  51
  52 import static java.util.Arrays.asList;
  53 import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
  54 import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
  55 import static org.assertj.core.api.Assertions.assertThat;
  56 import static org.assertj.core.api.Assertions.entry;
  57 import static org.assertj.core.groups.Tuple.tuple;
  58 import static org.mockito.Mockito.mock;
  59 import static org.mockito.Mockito.spy;
  60 import static org.mockito.Mockito.when;
  61 import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY;
  62 import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
  63 import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
  64 import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY;
  65 import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI;
  66 import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM;
  67 import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY;
  68 import static org.sonar.core.platform.EditionProvider.Edition.DEVELOPER;
  69 import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE;
  70 import static org.sonar.db.component.BranchType.BRANCH;
  71 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY;
  72 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY;
  73
  74 public class TelemetryDataLoaderImplTest {
  75   private final static Long NOW = 100_000_000L;
  76   private final TestSystem2 system2 = new TestSystem2().setNow(NOW);
  77
  78   @Rule
  79   public DbTester db = DbTester.create(system2);
  80
  81   private final FakeServer server = new FakeServer();
  82   private final PluginRepository pluginRepository = mock(PluginRepository.class);
  83   private final Configuration configuration = mock(Configuration.class);
  84   private final PlatformEditionProvider editionProvider = mock(PlatformEditionProvider.class);
  85   private final DockerSupport dockerSupport = mock(DockerSupport.class);
  86   private final InternalProperties internalProperties = spy(new MapInternalProperties());
  87   private final LicenseReader licenseReader = mock(LicenseReader.class);
  88
  89   private final TelemetryDataLoader communityUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
  90       internalProperties, configuration, dockerSupport, null);
  91   private final TelemetryDataLoader commercialUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
  92       internalProperties, configuration, dockerSupport, licenseReader);
  93
  94   @Test
  95   public void send_telemetry_data() {
  96     String serverId = "AU-TpxcB-iU5OvuD2FL7";
  97     String version = "7.5.4";
  98     Long analysisDate = 1L;
  99     Long lastConnectionDate = 5L;
 100
 101     server.setId(serverId);
 102     server.setVersion(version);
 103     List<PluginInfo> plugins = asList(newPlugin("java", "4.12.0.11033"), newPlugin("scmgit", "1.2"), new PluginInfo("other"));
 104     when(pluginRepository.getPluginInfos()).thenReturn(plugins);
 105     when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER));
 106
 107     int activeUserCount = 3;
 108     List<UserDto> activeUsers = IntStream.range(0, activeUserCount).mapToObj(i -> db.users().insertUser(
 109       u -> u.setExternalIdentityProvider("provider" + i).setLastSonarlintConnectionDate(i * 2L)))
 110       .collect(Collectors.toList());
 111
 112     // update last connection
 113     activeUsers.forEach(u -> db.users().updateLastConnectionDate(u, 5L));
 114
 115     UserDto inactiveUser = db.users().insertUser(u -> u.setActive(false).setExternalIdentityProvider("provider0"));
 116
 117     MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY));
 118     MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY));
 119     MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY));
 120     MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY));
 121
 122     ComponentDto project1 = db.components().insertPrivateProject();
 123     db.measures().insertLiveMeasure(project1, lines, m -> m.setValue(110d));
 124     db.measures().insertLiveMeasure(project1, ncloc, m -> m.setValue(110d));
 125     db.measures().insertLiveMeasure(project1, coverage, m -> m.setValue(80d));
 126     db.measures().insertLiveMeasure(project1, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10"));
 127
 128     ComponentDto project2 = db.components().insertPrivateProject();
 129     db.measures().insertLiveMeasure(project2, lines, m -> m.setValue(200d));
 130     db.measures().insertLiveMeasure(project2, ncloc, m -> m.setValue(200d));
 131     db.measures().insertLiveMeasure(project2, coverage, m -> m.setValue(80d));
 132     db.measures().insertLiveMeasure(project2, nclocDistrib, m -> m.setValue(null).setData("java=180;js=20"));
 133
 134     SnapshotDto project1Analysis = db.components().insertSnapshot(project1, t -> t.setLast(true).setBuildDate(analysisDate));
 135     SnapshotDto project2Analysis = db.components().insertSnapshot(project2, t -> t.setLast(true).setBuildDate(analysisDate));
 136     db.measures().insertMeasure(project1, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10"));
 137     db.measures().insertMeasure(project2, project2Analysis, nclocDistrib, m -> m.setData("java=180;js=20"));
 138
 139     insertAnalysisProperty(project1Analysis, "prop-uuid-1", SONAR_ANALYSIS_DETECTEDCI, "ci-1");
 140     insertAnalysisProperty(project2Analysis, "prop-uuid-2", SONAR_ANALYSIS_DETECTEDCI, "ci-2");
 141     insertAnalysisProperty(project1Analysis, "prop-uuid-3", SONAR_ANALYSIS_DETECTEDSCM, "scm-1");
 142     insertAnalysisProperty(project2Analysis, "prop-uuid-4", SONAR_ANALYSIS_DETECTEDSCM, "scm-2");
 143
 144     // alm
 145     db.almSettings().insertAzureAlmSetting();
 146     db.almSettings().insertGitHubAlmSetting();
 147     AlmSettingDto almSettingDto = db.almSettings().insertAzureAlmSetting(a -> a.setUrl("https://dev.azure.com"));
 148     AlmSettingDto gitHubAlmSetting = db.almSettings().insertGitHubAlmSetting(a -> a.setUrl("https://api.github.com"));
 149     db.almSettings().insertAzureProjectAlmSetting(almSettingDto, db.components().getProjectDto(project1));
 150     db.almSettings().insertGitlabProjectAlmSetting(gitHubAlmSetting, db.components().getProjectDto(project2));
 151
 152     TelemetryData data = communityUnderTest.load();
 153     assertThat(data.getServerId()).isEqualTo(serverId);
 154     assertThat(data.getVersion()).isEqualTo(version);
 155     assertThat(data.getEdition()).contains(DEVELOPER);
 156     assertDatabaseMetadata(data.getDatabase());
 157     assertThat(data.getPlugins()).containsOnly(
 158       entry("java", "4.12.0.11033"), entry("scmgit", "1.2"), entry("other", "undefined"));
 159     assertThat(data.isInDocker()).isFalse();
 160     assertThat(data.getExternalAuthenticationProviders()).containsExactlyInAnyOrder("provider0", "provider1", "provider2");
 161
 162     assertThat(data.getUserTelemetries())
 163       .extracting(UserTelemetryDto::getUuid, UserTelemetryDto::getLastConnectionDate, UserTelemetryDto::getLastSonarlintConnectionDate, UserTelemetryDto::isActive)
 164       .containsExactlyInAnyOrder(
 165         tuple(activeUsers.get(0).getUuid(), lastConnectionDate, activeUsers.get(0).getLastSonarlintConnectionDate(), true),
 166         tuple(activeUsers.get(1).getUuid(), lastConnectionDate, activeUsers.get(1).getLastSonarlintConnectionDate(), true),
 167         tuple(activeUsers.get(2).getUuid(), lastConnectionDate, activeUsers.get(2).getLastSonarlintConnectionDate(), true),
 168         tuple(inactiveUser.getUuid(), null, inactiveUser.getLastSonarlintConnectionDate(), false));
 169     assertThat(data.getProjects())
 170       .extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc, TelemetryData.Project::getLastAnalysis)
 171       .containsExactlyInAnyOrder(
 172         tuple(project1.uuid(), "java", 70L, analysisDate),
 173         tuple(project1.uuid(), "js", 30L, analysisDate),
 174         tuple(project1.uuid(), "kotlin", 10L, analysisDate),
 175         tuple(project2.uuid(), "java", 180L, analysisDate),
 176         tuple(project2.uuid(), "js", 20L, analysisDate));
 177     assertThat(data.getProjectStatistics())
 178       .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount,
 179         TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi, TelemetryData.ProjectStatistics::getAlm)
 180       .containsExactlyInAnyOrder(
 181         tuple(1L, 0L, "scm-1", "ci-1", "azure_devops_cloud"),
 182         tuple(1L, 0L, "scm-2", "ci-2", "github_cloud"));
 183   }
 184
 185   private void assertDatabaseMetadata(TelemetryData.Database database) {
 186     try (DbSession dbSession = db.getDbClient().openSession(false)) {
 187       DatabaseMetaData metadata = dbSession.getConnection().getMetaData();
 188       assertThat(database.getName()).isEqualTo("H2");
 189       assertThat(database.getVersion()).isEqualTo(metadata.getDatabaseProductVersion());
 190     } catch (SQLException e) {
 191       throw new RuntimeException(e);
 192     }
 193   }
 194
 195   @Test
 196   public void take_largest_branch_snapshot_project_data() {
 197     server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4");
 198
 199     MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY));
 200     MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY));
 201     MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY));
 202     MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY));
 203
 204     ComponentDto project = db.components().insertPublicProject();
 205     db.measures().insertLiveMeasure(project, lines, m -> m.setValue(110d));
 206     db.measures().insertLiveMeasure(project, ncloc, m -> m.setValue(110d));
 207     db.measures().insertLiveMeasure(project, coverage, m -> m.setValue(80d));
 208     db.measures().insertLiveMeasure(project, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10"));
 209
 210     ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setBranchType(BRANCH));
 211     db.measures().insertLiveMeasure(branch, lines, m -> m.setValue(180d));
 212     db.measures().insertLiveMeasure(branch, ncloc, m -> m.setValue(180d));
 213     db.measures().insertLiveMeasure(branch, coverage, m -> m.setValue(80d));
 214     db.measures().insertLiveMeasure(branch, nclocDistrib, m -> m.setValue(null).setData("java=100;js=50;kotlin=30"));
 215
 216     SnapshotDto project1Analysis = db.components().insertSnapshot(project, t -> t.setLast(true));
 217     SnapshotDto project2Analysis = db.components().insertSnapshot(branch, t -> t.setLast(true));
 218     db.measures().insertMeasure(project, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10"));
 219     db.measures().insertMeasure(branch, project2Analysis, nclocDistrib, m -> m.setData("java=100;js=50;kotlin=30"));
 220
 221     TelemetryData data = communityUnderTest.load();
 222
 223     assertThat(data.getProjects()).extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc)
 224       .containsExactlyInAnyOrder(
 225         tuple(project.uuid(), "java", 100L),
 226         tuple(project.uuid(), "js", 50L),
 227         tuple(project.uuid(), "kotlin", 30L));
 228     assertThat(data.getProjectStatistics())
 229       .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount,
 230         TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
 231       .containsExactlyInAnyOrder(
 232         tuple(2L, 0L, "undetected", "undetected"));
 233   }
 234
 235   @Test
 236   public void data_contains_no_license_type_on_community_edition() {
 237     TelemetryData data = communityUnderTest.load();
 238
 239     assertThat(data.getLicenseType()).isEmpty();
 240   }
 241
 242   @Test
 243   public void data_contains_no_license_type_on_commercial_edition_if_no_license() {
 244     when(licenseReader.read()).thenReturn(Optional.empty());
 245
 246     TelemetryData data = commercialUnderTest.load();
 247
 248     assertThat(data.getLicenseType()).isEmpty();
 249   }
 250
 251   @Test
 252   public void data_contains_weekly_count_sonarlint_users() {
 253     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 100_000L));
 254     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW));
 255     // these don't count
 256     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 1_000_000_000L));
 257     db.users().insertUser();
 258
 259     TelemetryData data = communityUnderTest.load();
 260     assertThat(data.getUserTelemetries())
 261       .hasSize(4);
 262   }
 263
 264   @Test
 265   public void data_has_license_type_on_commercial_edition_if_no_license() {
 266     String licenseType = randomAlphabetic(12);
 267     LicenseReader.License license = mock(LicenseReader.License.class);
 268     when(license.getType()).thenReturn(licenseType);
 269     when(licenseReader.read()).thenReturn(Optional.of(license));
 270
 271     TelemetryData data = commercialUnderTest.load();
 272
 273     assertThat(data.getLicenseType()).contains(licenseType);
 274   }
 275
 276   @Test
 277   public void send_server_id_and_version() {
 278     String id = randomAlphanumeric(40);
 279     String version = randomAlphanumeric(10);
 280     server.setId(id);
 281     server.setVersion(version);
 282
 283     TelemetryData data = communityUnderTest.load();
 284     assertThat(data.getServerId()).isEqualTo(id);
 285     assertThat(data.getVersion()).isEqualTo(version);
 286
 287     data = commercialUnderTest.load();
 288     assertThat(data.getServerId()).isEqualTo(id);
 289     assertThat(data.getVersion()).isEqualTo(version);
 290   }
 291
 292   @Test
 293   public void send_server_installation_date_and_installation_version() {
 294     String installationVersion = "7.9.BEST.LTS.EVER";
 295     Long installationDate = 1546300800000L; // 2019/01/01
 296     internalProperties.write(InternalProperties.INSTALLATION_DATE, String.valueOf(installationDate));
 297     internalProperties.write(InternalProperties.INSTALLATION_VERSION, installationVersion);
 298
 299     TelemetryData data = communityUnderTest.load();
 300
 301     assertThat(data.getInstallationDate()).isEqualTo(installationDate);
 302     assertThat(data.getInstallationVersion()).isEqualTo(installationVersion);
 303   }
 304
 305   @Test
 306   public void do_not_send_server_installation_details_if_missing_property() {
 307     TelemetryData data = communityUnderTest.load();
 308     assertThat(data.getInstallationDate()).isNull();
 309     assertThat(data.getInstallationVersion()).isNull();
 310
 311     data = commercialUnderTest.load();
 312     assertThat(data.getInstallationDate()).isNull();
 313     assertThat(data.getInstallationVersion()).isNull();
 314   }
 315
 316   @Test
 317   public void send_unanalyzed_languages_flags_when_edition_is_community() {
 318     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
 319     MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY));
 320     MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY));
 321     ComponentDto project1 = db.components().insertPublicProject();
 322     ComponentDto project2 = db.components().insertPublicProject();
 323     db.measures().insertLiveMeasure(project1, unanalyzedC);
 324     db.measures().insertLiveMeasure(project2, unanalyzedC);
 325     db.measures().insertLiveMeasure(project2, unanalyzedCpp);
 326
 327     TelemetryData data = communityUnderTest.load();
 328
 329     assertThat(data.hasUnanalyzedC().get()).isTrue();
 330     assertThat(data.hasUnanalyzedCpp().get()).isTrue();
 331   }
 332
 333   @Test
 334   public void do_not_send_unanalyzed_languages_flags_when_edition_is_not_community() {
 335     when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER));
 336     MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY));
 337     MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY));
 338     ComponentDto project1 = db.components().insertPublicProject();
 339     ComponentDto project2 = db.components().insertPublicProject();
 340     db.measures().insertLiveMeasure(project1, unanalyzedC);
 341     db.measures().insertLiveMeasure(project2, unanalyzedCpp);
 342
 343     TelemetryData data = communityUnderTest.load();
 344
 345     assertThat(data.hasUnanalyzedC()).isEmpty();
 346     assertThat(data.hasUnanalyzedCpp()).isEmpty();
 347   }
 348
 349   @Test
 350   public void unanalyzed_languages_flags_are_set_to_false_when_no_unanalyzed_languages_and_edition_is_community() {
 351     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
 352
 353     TelemetryData data = communityUnderTest.load();
 354
 355     assertThat(data.hasUnanalyzedC().get()).isFalse();
 356     assertThat(data.hasUnanalyzedCpp().get()).isFalse();
 357   }
 358
 359   @Test
 360   public void populate_security_custom_config_for_languages_on_enterprise() {
 361     when(editionProvider.get()).thenReturn(Optional.of(ENTERPRISE));
 362
 363     when(configuration.get("sonar.security.config.javasecurity")).thenReturn(Optional.of("{}"));
 364     when(configuration.get("sonar.security.config.phpsecurity")).thenReturn(Optional.of("{}"));
 365     when(configuration.get("sonar.security.config.pythonsecurity")).thenReturn(Optional.of("{}"));
 366     when(configuration.get("sonar.security.config.roslyn.sonaranalyzer.security.cs")).thenReturn(Optional.of("{}"));
 367
 368     TelemetryData data = commercialUnderTest.load();
 369
 370     assertThat(data.getCustomSecurityConfigs())
 371       .containsExactlyInAnyOrder("java", "php", "python", "csharp");
 372   }
 373
 374   @Test
 375   public void skip_security_custom_config_on_community() {
 376     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
 377
 378     TelemetryData data = communityUnderTest.load();
 379
 380     assertThat(data.getCustomSecurityConfigs()).isEmpty();
 381   }
 382
 383   @Test
 384   public void undetected_alm_ci_slm_data() {
 385     server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4");
 386     db.components().insertPublicProject();
 387     TelemetryData data = communityUnderTest.load();
 388     assertThat(data.getProjectStatistics())
 389       .extracting(TelemetryData.ProjectStatistics::getAlm, TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
 390       .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected"));
 391   }
 392
 393   private PluginInfo newPlugin(String key, String version) {
 394     return new PluginInfo(key)
 395       .setVersion(Version.create(version));
 396   }
 397
 398   private void insertAnalysisProperty(SnapshotDto snapshotDto, String uuid, String key, String value) {
 399     db.getDbClient().analysisPropertiesDao().insert(db.getSession(), new AnalysisPropertyDto()
 400       .setUuid(uuid)
 401       .setAnalysisUuid(snapshotDto.getUuid())
 402       .setKey(key)
 403       .setValue(value)
 404       .setCreatedAt(1L));
 405   }
 406
 407 }