source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2023 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.server.telemetry;
  21
  22 import com.tngtech.java.junit.dataprovider.DataProvider;
  23 import com.tngtech.java.junit.dataprovider.DataProviderRunner;
  24 import com.tngtech.java.junit.dataprovider.UseDataProvider;
  25 import java.sql.DatabaseMetaData;
  26 import java.sql.SQLException;
  27 import java.util.List;
  28 import java.util.Optional;
  29 import java.util.Set;
  30 import java.util.function.Consumer;
  31 import java.util.function.Function;
  32 import java.util.stream.Collectors;
  33 import java.util.stream.IntStream;
  34 import org.junit.Rule;
  35 import org.junit.Test;
  36 import org.junit.runner.RunWith;
  37 import org.sonar.api.config.Configuration;
  38 import org.sonar.api.impl.utils.TestSystem2;
  39 import org.sonar.core.platform.PlatformEditionProvider;
  40 import org.sonar.core.platform.PluginInfo;
  41 import org.sonar.core.platform.PluginRepository;
  42 import org.sonar.db.DbSession;
  43 import org.sonar.db.DbTester;
  44 import org.sonar.db.alm.setting.AlmSettingDto;
  45 import org.sonar.db.component.AnalysisPropertyDto;
  46 import org.sonar.db.component.ComponentDto;
  47 import org.sonar.db.component.SnapshotDto;
  48 import org.sonar.db.metric.MetricDto;
  49 import org.sonar.db.user.UserDbTester;
  50 import org.sonar.db.user.UserDto;
  51 import org.sonar.db.user.UserTelemetryDto;
  52 import org.sonar.server.platform.DockerSupport;
  53 import org.sonar.server.property.InternalProperties;
  54 import org.sonar.server.property.MapInternalProperties;
  55 import org.sonar.updatecenter.common.Version;
  56
  57 import static java.util.Arrays.asList;
  58 import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
  59 import static org.assertj.core.api.Assertions.assertThat;
  60 import static org.assertj.core.api.Assertions.entry;
  61 import static org.assertj.core.groups.Tuple.tuple;
  62 import static org.mockito.Mockito.mock;
  63 import static org.mockito.Mockito.spy;
  64 import static org.mockito.Mockito.when;
  65 import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY;
  66 import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
  67 import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
  68 import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY;
  69 import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI;
  70 import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM;
  71 import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY;
  72 import static org.sonar.core.platform.EditionProvider.Edition.DEVELOPER;
  73 import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE;
  74 import static org.sonar.db.component.BranchType.BRANCH;
  75 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY;
  76 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY;
  77 import static org.sonar.server.telemetry.TelemetryDataLoaderImpl.SCIM_PROPERTY_ENABLED;
  78
  79 @RunWith(DataProviderRunner.class)
  80 public class TelemetryDataLoaderImplTest {
  81   private final static Long NOW = 100_000_000L;
  82   private final TestSystem2 system2 = new TestSystem2().setNow(NOW);
  83
  84   @Rule
  85   public DbTester db = DbTester.create(system2);
  86
  87   private final FakeServer server = new FakeServer();
  88   private final PluginRepository pluginRepository = mock(PluginRepository.class);
  89   private final Configuration configuration = mock(Configuration.class);
  90   private final PlatformEditionProvider editionProvider = mock(PlatformEditionProvider.class);
  91   private final DockerSupport dockerSupport = mock(DockerSupport.class);
  92   private final InternalProperties internalProperties = spy(new MapInternalProperties());
  93
  94   private final TelemetryDataLoader communityUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
  95       internalProperties, configuration, dockerSupport);
  96   private final TelemetryDataLoader commercialUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
  97       internalProperties, configuration, dockerSupport);
  98
  99   @Test
 100   public void send_telemetry_data() {
 101     String serverId = "AU-TpxcB-iU5OvuD2FL7";
 102     String version = "7.5.4";
 103     Long analysisDate = 1L;
 104     Long lastConnectionDate = 5L;
 105
 106     server.setId(serverId);
 107     server.setVersion(version);
 108     List<PluginInfo> plugins = asList(newPlugin("java", "4.12.0.11033"), newPlugin("scmgit", "1.2"), new PluginInfo("other"));
 109     when(pluginRepository.getPluginInfos()).thenReturn(plugins);
 110     when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER));
 111
 112     List<UserDto> activeUsers = composeActiveUsers(3);
 113
 114     // update last connection
 115     activeUsers.forEach(u -> db.users().updateLastConnectionDate(u, 5L));
 116
 117     UserDto inactiveUser = db.users().insertUser(u -> u.setActive(false).setExternalIdentityProvider("provider0"));
 118
 119     MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY));
 120     MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY));
 121     MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY));
 122     MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY));
 123
 124     ComponentDto project1 = db.components().insertPrivateProject();
 125     db.measures().insertLiveMeasure(project1, lines, m -> m.setValue(110d));
 126     db.measures().insertLiveMeasure(project1, ncloc, m -> m.setValue(110d));
 127     db.measures().insertLiveMeasure(project1, coverage, m -> m.setValue(80d));
 128     db.measures().insertLiveMeasure(project1, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10"));
 129
 130     ComponentDto project2 = db.components().insertPrivateProject();
 131     db.measures().insertLiveMeasure(project2, lines, m -> m.setValue(200d));
 132     db.measures().insertLiveMeasure(project2, ncloc, m -> m.setValue(200d));
 133     db.measures().insertLiveMeasure(project2, coverage, m -> m.setValue(80d));
 134     db.measures().insertLiveMeasure(project2, nclocDistrib, m -> m.setValue(null).setData("java=180;js=20"));
 135
 136     SnapshotDto project1Analysis = db.components().insertSnapshot(project1, t -> t.setLast(true).setBuildDate(analysisDate));
 137     SnapshotDto project2Analysis = db.components().insertSnapshot(project2, t -> t.setLast(true).setBuildDate(analysisDate));
 138     db.measures().insertMeasure(project1, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10"));
 139     db.measures().insertMeasure(project2, project2Analysis, nclocDistrib, m -> m.setData("java=180;js=20"));
 140
 141     insertAnalysisProperty(project1Analysis, "prop-uuid-1", SONAR_ANALYSIS_DETECTEDCI, "ci-1");
 142     insertAnalysisProperty(project2Analysis, "prop-uuid-2", SONAR_ANALYSIS_DETECTEDCI, "ci-2");
 143     insertAnalysisProperty(project1Analysis, "prop-uuid-3", SONAR_ANALYSIS_DETECTEDSCM, "scm-1");
 144     insertAnalysisProperty(project2Analysis, "prop-uuid-4", SONAR_ANALYSIS_DETECTEDSCM, "scm-2");
 145
 146     // alm
 147     db.almSettings().insertAzureAlmSetting();
 148     db.almSettings().insertGitHubAlmSetting();
 149     AlmSettingDto almSettingDto = db.almSettings().insertAzureAlmSetting(a -> a.setUrl("https://dev.azure.com"));
 150     AlmSettingDto gitHubAlmSetting = db.almSettings().insertGitHubAlmSetting(a -> a.setUrl("https://api.github.com"));
 151     db.almSettings().insertAzureProjectAlmSetting(almSettingDto, db.components().getProjectDto(project1));
 152     db.almSettings().insertGitlabProjectAlmSetting(gitHubAlmSetting, db.components().getProjectDto(project2));
 153
 154     TelemetryData data = communityUnderTest.load();
 155     assertThat(data.getServerId()).isEqualTo(serverId);
 156     assertThat(data.getVersion()).isEqualTo(version);
 157     assertThat(data.getEdition()).contains(DEVELOPER);
 158     assertThat(data.getMessageSequenceNumber()).isOne();
 159     assertDatabaseMetadata(data.getDatabase());
 160     assertThat(data.getPlugins()).containsOnly(
 161       entry("java", "4.12.0.11033"), entry("scmgit", "1.2"), entry("other", "undefined"));
 162     assertThat(data.isInDocker()).isFalse();
 163
 164     assertThat(data.getUserTelemetries())
 165       .extracting(UserTelemetryDto::getUuid, UserTelemetryDto::getLastConnectionDate, UserTelemetryDto::getLastSonarlintConnectionDate, UserTelemetryDto::isActive)
 166       .containsExactlyInAnyOrder(
 167         tuple(activeUsers.get(0).getUuid(), lastConnectionDate, activeUsers.get(0).getLastSonarlintConnectionDate(), true),
 168         tuple(activeUsers.get(1).getUuid(), lastConnectionDate, activeUsers.get(1).getLastSonarlintConnectionDate(), true),
 169         tuple(activeUsers.get(2).getUuid(), lastConnectionDate, activeUsers.get(2).getLastSonarlintConnectionDate(), true),
 170         tuple(inactiveUser.getUuid(), null, inactiveUser.getLastSonarlintConnectionDate(), false));
 171     assertThat(data.getProjects())
 172       .extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc, TelemetryData.Project::getLastAnalysis)
 173       .containsExactlyInAnyOrder(
 174         tuple(project1.uuid(), "java", 70L, analysisDate),
 175         tuple(project1.uuid(), "js", 30L, analysisDate),
 176         tuple(project1.uuid(), "kotlin", 10L, analysisDate),
 177         tuple(project2.uuid(), "java", 180L, analysisDate),
 178         tuple(project2.uuid(), "js", 20L, analysisDate));
 179     assertThat(data.getProjectStatistics())
 180       .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount,
 181         TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi, TelemetryData.ProjectStatistics::getDevopsPlatform)
 182       .containsExactlyInAnyOrder(
 183         tuple(1L, 0L, "scm-1", "ci-1", "azure_devops_cloud"),
 184         tuple(1L, 0L, "scm-2", "ci-2", "github_cloud"));
 185   }
 186
 187   private List<UserDto> composeActiveUsers(int count) {
 188     UserDbTester userDbTester = db.users();
 189     Function<Integer, Consumer<UserDto>> userConfigurator = index -> user -> user.setExternalIdentityProvider("provider" + index).setLastSonarlintConnectionDate(index * 2L);
 190
 191     return IntStream
 192       .rangeClosed(1, count)
 193       .mapToObj(userConfigurator::apply)
 194       .map(userDbTester::insertUser)
 195       .collect(Collectors.toList());
 196   }
 197
 198   private void assertDatabaseMetadata(TelemetryData.Database database) {
 199     try (DbSession dbSession = db.getDbClient().openSession(false)) {
 200       DatabaseMetaData metadata = dbSession.getConnection().getMetaData();
 201       assertThat(database.getName()).isEqualTo("H2");
 202       assertThat(database.getVersion()).isEqualTo(metadata.getDatabaseProductVersion());
 203     } catch (SQLException e) {
 204       throw new RuntimeException(e);
 205     }
 206   }
 207
 208   @Test
 209   public void take_largest_branch_snapshot_project_data() {
 210     server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4");
 211
 212     MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY));
 213     MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY));
 214     MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY));
 215     MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY));
 216
 217     ComponentDto project = db.components().insertPublicProject();
 218     db.measures().insertLiveMeasure(project, lines, m -> m.setValue(110d));
 219     db.measures().insertLiveMeasure(project, ncloc, m -> m.setValue(110d));
 220     db.measures().insertLiveMeasure(project, coverage, m -> m.setValue(80d));
 221     db.measures().insertLiveMeasure(project, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10"));
 222
 223     ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setBranchType(BRANCH));
 224     db.measures().insertLiveMeasure(branch, lines, m -> m.setValue(180d));
 225     db.measures().insertLiveMeasure(branch, ncloc, m -> m.setValue(180d));
 226     db.measures().insertLiveMeasure(branch, coverage, m -> m.setValue(80d));
 227     db.measures().insertLiveMeasure(branch, nclocDistrib, m -> m.setValue(null).setData("java=100;js=50;kotlin=30"));
 228
 229     SnapshotDto project1Analysis = db.components().insertSnapshot(project, t -> t.setLast(true));
 230     SnapshotDto project2Analysis = db.components().insertSnapshot(branch, t -> t.setLast(true));
 231     db.measures().insertMeasure(project, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10"));
 232     db.measures().insertMeasure(branch, project2Analysis, nclocDistrib, m -> m.setData("java=100;js=50;kotlin=30"));
 233
 234     TelemetryData data = communityUnderTest.load();
 235
 236     assertThat(data.getProjects()).extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc)
 237       .containsExactlyInAnyOrder(
 238         tuple(project.uuid(), "java", 100L),
 239         tuple(project.uuid(), "js", 50L),
 240         tuple(project.uuid(), "kotlin", 30L));
 241     assertThat(data.getProjectStatistics())
 242       .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount,
 243         TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
 244       .containsExactlyInAnyOrder(
 245         tuple(2L, 0L, "undetected", "undetected"));
 246   }
 247
 248   @Test
 249   public void data_contains_weekly_count_sonarlint_users() {
 250     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 100_000L));
 251     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW));
 252     // these don't count
 253     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 1_000_000_000L));
 254     db.users().insertUser();
 255
 256     TelemetryData data = communityUnderTest.load();
 257     assertThat(data.getUserTelemetries())
 258       .hasSize(4);
 259   }
 260
 261   @Test
 262   public void send_server_id_and_version() {
 263     String id = randomAlphanumeric(40);
 264     String version = randomAlphanumeric(10);
 265     server.setId(id);
 266     server.setVersion(version);
 267
 268     TelemetryData data = communityUnderTest.load();
 269     assertThat(data.getServerId()).isEqualTo(id);
 270     assertThat(data.getVersion()).isEqualTo(version);
 271
 272     data = commercialUnderTest.load();
 273     assertThat(data.getServerId()).isEqualTo(id);
 274     assertThat(data.getVersion()).isEqualTo(version);
 275   }
 276
 277   @Test
 278   public void send_server_installation_date_and_installation_version() {
 279     String installationVersion = "7.9.BEST.LTS.EVER";
 280     Long installationDate = 1546300800000L; // 2019/01/01
 281     internalProperties.write(InternalProperties.INSTALLATION_DATE, String.valueOf(installationDate));
 282     internalProperties.write(InternalProperties.INSTALLATION_VERSION, installationVersion);
 283
 284     TelemetryData data = communityUnderTest.load();
 285
 286     assertThat(data.getInstallationDate()).isEqualTo(installationDate);
 287     assertThat(data.getInstallationVersion()).isEqualTo(installationVersion);
 288   }
 289
 290   @Test
 291   public void send_correct_sequence_number() {
 292     internalProperties.write(TelemetryDaemon.I_PROP_MESSAGE_SEQUENCE, "10");
 293     TelemetryData data = communityUnderTest.load();
 294     assertThat(data.getMessageSequenceNumber()).isEqualTo(11L);
 295   }
 296
 297   @Test
 298   public void do_not_send_server_installation_details_if_missing_property() {
 299     TelemetryData data = communityUnderTest.load();
 300     assertThat(data.getInstallationDate()).isNull();
 301     assertThat(data.getInstallationVersion()).isNull();
 302
 303     data = commercialUnderTest.load();
 304     assertThat(data.getInstallationDate()).isNull();
 305     assertThat(data.getInstallationVersion()).isNull();
 306   }
 307
 308   @Test
 309   public void send_unanalyzed_languages_flags_when_edition_is_community() {
 310     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
 311     MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY));
 312     MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY));
 313     ComponentDto project1 = db.components().insertPublicProject();
 314     ComponentDto project2 = db.components().insertPublicProject();
 315     db.measures().insertLiveMeasure(project1, unanalyzedC);
 316     db.measures().insertLiveMeasure(project2, unanalyzedC);
 317     db.measures().insertLiveMeasure(project2, unanalyzedCpp);
 318
 319     TelemetryData data = communityUnderTest.load();
 320
 321     assertThat(data.getProjectStatistics())
 322       .extracting(TelemetryData.ProjectStatistics::hasUnanalyzedC, TelemetryData.ProjectStatistics::hasUnanalyzedCpp)
 323       .containsExactlyInAnyOrder(tuple(Optional.of(true), Optional.of(true)), tuple(Optional.of(true), Optional.of(false)));
 324   }
 325
 326   @Test
 327   public void do_not_send_unanalyzed_languages_flags_when_edition_is_not_community() {
 328     when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER));
 329     MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY));
 330     MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY));
 331     ComponentDto project1 = db.components().insertPublicProject();
 332     ComponentDto project2 = db.components().insertPublicProject();
 333     db.measures().insertLiveMeasure(project1, unanalyzedC);
 334     db.measures().insertLiveMeasure(project2, unanalyzedCpp);
 335
 336     TelemetryData data = communityUnderTest.load();
 337
 338     assertThat(data.getProjectStatistics())
 339       .extracting(TelemetryData.ProjectStatistics::hasUnanalyzedC, TelemetryData.ProjectStatistics::hasUnanalyzedCpp)
 340       .containsExactlyInAnyOrder(tuple(Optional.empty(), Optional.empty()), tuple(Optional.empty(), Optional.empty()));
 341   }
 342
 343   @Test
 344   public void populate_security_custom_config_for_languages_on_enterprise() {
 345     when(editionProvider.get()).thenReturn(Optional.of(ENTERPRISE));
 346
 347     when(configuration.get("sonar.security.config.javasecurity")).thenReturn(Optional.of("{}"));
 348     when(configuration.get("sonar.security.config.phpsecurity")).thenReturn(Optional.of("{}"));
 349     when(configuration.get("sonar.security.config.pythonsecurity")).thenReturn(Optional.of("{}"));
 350     when(configuration.get("sonar.security.config.roslyn.sonaranalyzer.security.cs")).thenReturn(Optional.of("{}"));
 351
 352     TelemetryData data = commercialUnderTest.load();
 353
 354     assertThat(data.getCustomSecurityConfigs())
 355       .containsExactlyInAnyOrder("java", "php", "python", "csharp");
 356   }
 357
 358   @Test
 359   public void skip_security_custom_config_on_community() {
 360     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
 361
 362     TelemetryData data = communityUnderTest.load();
 363
 364     assertThat(data.getCustomSecurityConfigs()).isEmpty();
 365   }
 366
 367   @Test
 368   public void undetected_alm_ci_slm_data() {
 369     server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4");
 370     db.components().insertPublicProject();
 371     TelemetryData data = communityUnderTest.load();
 372     assertThat(data.getProjectStatistics())
 373       .extracting(TelemetryData.ProjectStatistics::getDevopsPlatform, TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
 374       .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected"));
 375   }
 376
 377   @Test
 378   @UseDataProvider("getScimFeatureStatues")
 379   public void detect_scim_feature_status(boolean isEnabled) {
 380     db.components().insertPublicProject();
 381     when(configuration.getBoolean(SCIM_PROPERTY_ENABLED)).thenReturn(Optional.of(isEnabled));
 382
 383     TelemetryData data = communityUnderTest.load();
 384
 385     assertThat(data.isScimEnabled()).isEqualTo(isEnabled);
 386   }
 387
 388   private PluginInfo newPlugin(String key, String version) {
 389     return new PluginInfo(key)
 390       .setVersion(Version.create(version));
 391   }
 392
 393   private void insertAnalysisProperty(SnapshotDto snapshotDto, String uuid, String key, String value) {
 394     db.getDbClient().analysisPropertiesDao().insert(db.getSession(), new AnalysisPropertyDto()
 395       .setUuid(uuid)
 396       .setAnalysisUuid(snapshotDto.getUuid())
 397       .setKey(key)
 398       .setValue(value)
 399       .setCreatedAt(1L));
 400   }
 401
 402   @DataProvider
 403   public static Set<Boolean> getScimFeatureStatues() {
 404     return Set.of(true, false);
 405   }
 406 }