]> source.dussan.org Git - sonarqube.git/blob
1e384cdeba629fb9c6054aee84b73219ef0b27c8
[sonarqube.git] /
1 /*
2  * SonarQube
3  * Copyright (C) 2009-2022 SonarSource SA
4  * mailto:info AT sonarsource DOT com
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 3 of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
19  */
20 package org.sonar.server.telemetry;
21
22 import com.tngtech.java.junit.dataprovider.DataProvider;
23 import com.tngtech.java.junit.dataprovider.DataProviderRunner;
24 import com.tngtech.java.junit.dataprovider.UseDataProvider;
25 import java.sql.DatabaseMetaData;
26 import java.sql.SQLException;
27 import java.util.List;
28 import java.util.Optional;
29 import java.util.Set;
30 import java.util.function.Consumer;
31 import java.util.function.Function;
32 import java.util.stream.Collectors;
33 import java.util.stream.IntStream;
34 import org.junit.Rule;
35 import org.junit.Test;
36 import org.junit.runner.RunWith;
37 import org.sonar.api.config.Configuration;
38 import org.sonar.api.impl.utils.TestSystem2;
39 import org.sonar.core.platform.PlatformEditionProvider;
40 import org.sonar.core.platform.PluginInfo;
41 import org.sonar.core.platform.PluginRepository;
42 import org.sonar.db.DbSession;
43 import org.sonar.db.DbTester;
44 import org.sonar.db.alm.setting.AlmSettingDto;
45 import org.sonar.db.component.AnalysisPropertyDto;
46 import org.sonar.db.component.ComponentDto;
47 import org.sonar.db.component.SnapshotDto;
48 import org.sonar.db.metric.MetricDto;
49 import org.sonar.db.user.UserDbTester;
50 import org.sonar.db.user.UserDto;
51 import org.sonar.db.user.UserTelemetryDto;
52 import org.sonar.server.platform.DockerSupport;
53 import org.sonar.server.property.InternalProperties;
54 import org.sonar.server.property.MapInternalProperties;
55 import org.sonar.updatecenter.common.Version;
56
57 import static java.util.Arrays.asList;
58 import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
59 import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
60 import static org.assertj.core.api.Assertions.assertThat;
61 import static org.assertj.core.api.Assertions.entry;
62 import static org.assertj.core.groups.Tuple.tuple;
63 import static org.mockito.Mockito.mock;
64 import static org.mockito.Mockito.spy;
65 import static org.mockito.Mockito.when;
66 import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY;
67 import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
68 import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
69 import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY;
70 import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI;
71 import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM;
72 import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY;
73 import static org.sonar.core.platform.EditionProvider.Edition.DEVELOPER;
74 import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE;
75 import static org.sonar.db.component.BranchType.BRANCH;
76 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY;
77 import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY;
78 import static org.sonar.server.telemetry.TelemetryDataLoaderImpl.SCIM_PROPERTY_ENABLED;
79
80 @RunWith(DataProviderRunner.class)
81 public class TelemetryDataLoaderImplTest {
82   private final static Long NOW = 100_000_000L;
83   private final TestSystem2 system2 = new TestSystem2().setNow(NOW);
84
85   @Rule
86   public DbTester db = DbTester.create(system2);
87
88   private final FakeServer server = new FakeServer();
89   private final PluginRepository pluginRepository = mock(PluginRepository.class);
90   private final Configuration configuration = mock(Configuration.class);
91   private final PlatformEditionProvider editionProvider = mock(PlatformEditionProvider.class);
92   private final DockerSupport dockerSupport = mock(DockerSupport.class);
93   private final InternalProperties internalProperties = spy(new MapInternalProperties());
94   private final LicenseReader licenseReader = mock(LicenseReader.class);
95
96   private final TelemetryDataLoader communityUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
97       internalProperties, configuration, dockerSupport, null);
98   private final TelemetryDataLoader commercialUnderTest = new TelemetryDataLoaderImpl(server, db.getDbClient(), pluginRepository, editionProvider,
99       internalProperties, configuration, dockerSupport, licenseReader);
100
101   @Test
102   public void send_telemetry_data() {
103     String serverId = "AU-TpxcB-iU5OvuD2FL7";
104     String version = "7.5.4";
105     Long analysisDate = 1L;
106     Long lastConnectionDate = 5L;
107
108     server.setId(serverId);
109     server.setVersion(version);
110     List<PluginInfo> plugins = asList(newPlugin("java", "4.12.0.11033"), newPlugin("scmgit", "1.2"), new PluginInfo("other"));
111     when(pluginRepository.getPluginInfos()).thenReturn(plugins);
112     when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER));
113
114     List<UserDto> activeUsers = composeActiveUsers(3);
115
116     // update last connection
117     activeUsers.forEach(u -> db.users().updateLastConnectionDate(u, 5L));
118
119     UserDto inactiveUser = db.users().insertUser(u -> u.setActive(false).setExternalIdentityProvider("provider0"));
120
121     MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY));
122     MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY));
123     MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY));
124     MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY));
125
126     ComponentDto project1 = db.components().insertPrivateProject();
127     db.measures().insertLiveMeasure(project1, lines, m -> m.setValue(110d));
128     db.measures().insertLiveMeasure(project1, ncloc, m -> m.setValue(110d));
129     db.measures().insertLiveMeasure(project1, coverage, m -> m.setValue(80d));
130     db.measures().insertLiveMeasure(project1, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10"));
131
132     ComponentDto project2 = db.components().insertPrivateProject();
133     db.measures().insertLiveMeasure(project2, lines, m -> m.setValue(200d));
134     db.measures().insertLiveMeasure(project2, ncloc, m -> m.setValue(200d));
135     db.measures().insertLiveMeasure(project2, coverage, m -> m.setValue(80d));
136     db.measures().insertLiveMeasure(project2, nclocDistrib, m -> m.setValue(null).setData("java=180;js=20"));
137
138     SnapshotDto project1Analysis = db.components().insertSnapshot(project1, t -> t.setLast(true).setBuildDate(analysisDate));
139     SnapshotDto project2Analysis = db.components().insertSnapshot(project2, t -> t.setLast(true).setBuildDate(analysisDate));
140     db.measures().insertMeasure(project1, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10"));
141     db.measures().insertMeasure(project2, project2Analysis, nclocDistrib, m -> m.setData("java=180;js=20"));
142
143     insertAnalysisProperty(project1Analysis, "prop-uuid-1", SONAR_ANALYSIS_DETECTEDCI, "ci-1");
144     insertAnalysisProperty(project2Analysis, "prop-uuid-2", SONAR_ANALYSIS_DETECTEDCI, "ci-2");
145     insertAnalysisProperty(project1Analysis, "prop-uuid-3", SONAR_ANALYSIS_DETECTEDSCM, "scm-1");
146     insertAnalysisProperty(project2Analysis, "prop-uuid-4", SONAR_ANALYSIS_DETECTEDSCM, "scm-2");
147
148     // alm
149     db.almSettings().insertAzureAlmSetting();
150     db.almSettings().insertGitHubAlmSetting();
151     AlmSettingDto almSettingDto = db.almSettings().insertAzureAlmSetting(a -> a.setUrl("https://dev.azure.com"));
152     AlmSettingDto gitHubAlmSetting = db.almSettings().insertGitHubAlmSetting(a -> a.setUrl("https://api.github.com"));
153     db.almSettings().insertAzureProjectAlmSetting(almSettingDto, db.components().getProjectDto(project1));
154     db.almSettings().insertGitlabProjectAlmSetting(gitHubAlmSetting, db.components().getProjectDto(project2));
155
156     TelemetryData data = communityUnderTest.load();
157     assertThat(data.getServerId()).isEqualTo(serverId);
158     assertThat(data.getVersion()).isEqualTo(version);
159     assertThat(data.getEdition()).contains(DEVELOPER);
160     assertDatabaseMetadata(data.getDatabase());
161     assertThat(data.getPlugins()).containsOnly(
162       entry("java", "4.12.0.11033"), entry("scmgit", "1.2"), entry("other", "undefined"));
163     assertThat(data.isInDocker()).isFalse();
164
165     assertThat(data.getUserTelemetries())
166       .extracting(UserTelemetryDto::getUuid, UserTelemetryDto::getLastConnectionDate, UserTelemetryDto::getLastSonarlintConnectionDate, UserTelemetryDto::isActive)
167       .containsExactlyInAnyOrder(
168         tuple(activeUsers.get(0).getUuid(), lastConnectionDate, activeUsers.get(0).getLastSonarlintConnectionDate(), true),
169         tuple(activeUsers.get(1).getUuid(), lastConnectionDate, activeUsers.get(1).getLastSonarlintConnectionDate(), true),
170         tuple(activeUsers.get(2).getUuid(), lastConnectionDate, activeUsers.get(2).getLastSonarlintConnectionDate(), true),
171         tuple(inactiveUser.getUuid(), null, inactiveUser.getLastSonarlintConnectionDate(), false));
172     assertThat(data.getProjects())
173       .extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc, TelemetryData.Project::getLastAnalysis)
174       .containsExactlyInAnyOrder(
175         tuple(project1.uuid(), "java", 70L, analysisDate),
176         tuple(project1.uuid(), "js", 30L, analysisDate),
177         tuple(project1.uuid(), "kotlin", 10L, analysisDate),
178         tuple(project2.uuid(), "java", 180L, analysisDate),
179         tuple(project2.uuid(), "js", 20L, analysisDate));
180     assertThat(data.getProjectStatistics())
181       .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount,
182         TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi, TelemetryData.ProjectStatistics::getDevopsPlatform)
183       .containsExactlyInAnyOrder(
184         tuple(1L, 0L, "scm-1", "ci-1", "azure_devops_cloud"),
185         tuple(1L, 0L, "scm-2", "ci-2", "github_cloud"));
186   }
187
188   private List<UserDto> composeActiveUsers(int count) {
189     UserDbTester userDbTester = db.users();
190     Function<Integer, Consumer<UserDto>> userConfigurator = index -> user -> user.setExternalIdentityProvider("provider" + index).setLastSonarlintConnectionDate(index * 2L);
191
192     return IntStream
193       .rangeClosed(1, count)
194       .mapToObj(userConfigurator::apply)
195       .map(userDbTester::insertUser)
196       .collect(Collectors.toList());
197   }
198
199   private void assertDatabaseMetadata(TelemetryData.Database database) {
200     try (DbSession dbSession = db.getDbClient().openSession(false)) {
201       DatabaseMetaData metadata = dbSession.getConnection().getMetaData();
202       assertThat(database.getName()).isEqualTo("H2");
203       assertThat(database.getVersion()).isEqualTo(metadata.getDatabaseProductVersion());
204     } catch (SQLException e) {
205       throw new RuntimeException(e);
206     }
207   }
208
209   @Test
210   public void take_largest_branch_snapshot_project_data() {
211     server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4");
212
213     MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY));
214     MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY));
215     MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY));
216     MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY));
217
218     ComponentDto project = db.components().insertPublicProject();
219     db.measures().insertLiveMeasure(project, lines, m -> m.setValue(110d));
220     db.measures().insertLiveMeasure(project, ncloc, m -> m.setValue(110d));
221     db.measures().insertLiveMeasure(project, coverage, m -> m.setValue(80d));
222     db.measures().insertLiveMeasure(project, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10"));
223
224     ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setBranchType(BRANCH));
225     db.measures().insertLiveMeasure(branch, lines, m -> m.setValue(180d));
226     db.measures().insertLiveMeasure(branch, ncloc, m -> m.setValue(180d));
227     db.measures().insertLiveMeasure(branch, coverage, m -> m.setValue(80d));
228     db.measures().insertLiveMeasure(branch, nclocDistrib, m -> m.setValue(null).setData("java=100;js=50;kotlin=30"));
229
230     SnapshotDto project1Analysis = db.components().insertSnapshot(project, t -> t.setLast(true));
231     SnapshotDto project2Analysis = db.components().insertSnapshot(branch, t -> t.setLast(true));
232     db.measures().insertMeasure(project, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10"));
233     db.measures().insertMeasure(branch, project2Analysis, nclocDistrib, m -> m.setData("java=100;js=50;kotlin=30"));
234
235     TelemetryData data = communityUnderTest.load();
236
237     assertThat(data.getProjects()).extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc)
238       .containsExactlyInAnyOrder(
239         tuple(project.uuid(), "java", 100L),
240         tuple(project.uuid(), "js", 50L),
241         tuple(project.uuid(), "kotlin", 30L));
242     assertThat(data.getProjectStatistics())
243       .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount,
244         TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
245       .containsExactlyInAnyOrder(
246         tuple(2L, 0L, "undetected", "undetected"));
247   }
248
249   @Test
250   public void data_contains_no_license_type_on_community_edition() {
251     TelemetryData data = communityUnderTest.load();
252
253     assertThat(data.getLicenseType()).isEmpty();
254   }
255
256   @Test
257   public void data_contains_no_license_type_on_commercial_edition_if_no_license() {
258     when(licenseReader.read()).thenReturn(Optional.empty());
259
260     TelemetryData data = commercialUnderTest.load();
261
262     assertThat(data.getLicenseType()).isEmpty();
263   }
264
265   @Test
266   public void data_contains_weekly_count_sonarlint_users() {
267     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 100_000L));
268     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW));
269     // these don't count
270     db.users().insertUser(c -> c.setLastSonarlintConnectionDate(NOW - 1_000_000_000L));
271     db.users().insertUser();
272
273     TelemetryData data = communityUnderTest.load();
274     assertThat(data.getUserTelemetries())
275       .hasSize(4);
276   }
277
278   @Test
279   public void data_has_license_type_on_commercial_edition_if_no_license() {
280     String licenseType = randomAlphabetic(12);
281     LicenseReader.License license = mock(LicenseReader.License.class);
282     when(license.getType()).thenReturn(licenseType);
283     when(licenseReader.read()).thenReturn(Optional.of(license));
284
285     TelemetryData data = commercialUnderTest.load();
286
287     assertThat(data.getLicenseType()).contains(licenseType);
288   }
289
290   @Test
291   public void send_server_id_and_version() {
292     String id = randomAlphanumeric(40);
293     String version = randomAlphanumeric(10);
294     server.setId(id);
295     server.setVersion(version);
296
297     TelemetryData data = communityUnderTest.load();
298     assertThat(data.getServerId()).isEqualTo(id);
299     assertThat(data.getVersion()).isEqualTo(version);
300
301     data = commercialUnderTest.load();
302     assertThat(data.getServerId()).isEqualTo(id);
303     assertThat(data.getVersion()).isEqualTo(version);
304   }
305
306   @Test
307   public void send_server_installation_date_and_installation_version() {
308     String installationVersion = "7.9.BEST.LTS.EVER";
309     Long installationDate = 1546300800000L; // 2019/01/01
310     internalProperties.write(InternalProperties.INSTALLATION_DATE, String.valueOf(installationDate));
311     internalProperties.write(InternalProperties.INSTALLATION_VERSION, installationVersion);
312
313     TelemetryData data = communityUnderTest.load();
314
315     assertThat(data.getInstallationDate()).isEqualTo(installationDate);
316     assertThat(data.getInstallationVersion()).isEqualTo(installationVersion);
317   }
318
319   @Test
320   public void do_not_send_server_installation_details_if_missing_property() {
321     TelemetryData data = communityUnderTest.load();
322     assertThat(data.getInstallationDate()).isNull();
323     assertThat(data.getInstallationVersion()).isNull();
324
325     data = commercialUnderTest.load();
326     assertThat(data.getInstallationDate()).isNull();
327     assertThat(data.getInstallationVersion()).isNull();
328   }
329
330   @Test
331   public void send_unanalyzed_languages_flags_when_edition_is_community() {
332     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
333     MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY));
334     MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY));
335     ComponentDto project1 = db.components().insertPublicProject();
336     ComponentDto project2 = db.components().insertPublicProject();
337     db.measures().insertLiveMeasure(project1, unanalyzedC);
338     db.measures().insertLiveMeasure(project2, unanalyzedC);
339     db.measures().insertLiveMeasure(project2, unanalyzedCpp);
340
341     TelemetryData data = communityUnderTest.load();
342
343     assertThat(data.getProjectStatistics())
344       .extracting(TelemetryData.ProjectStatistics::hasUnanalyzedC, TelemetryData.ProjectStatistics::hasUnanalyzedCpp)
345       .containsExactlyInAnyOrder(tuple(Optional.of(true), Optional.of(true)), tuple(Optional.of(true), Optional.of(false)));
346   }
347
348   @Test
349   public void do_not_send_unanalyzed_languages_flags_when_edition_is_not_community() {
350     when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER));
351     MetricDto unanalyzedC = db.measures().insertMetric(m -> m.setKey(UNANALYZED_C_KEY));
352     MetricDto unanalyzedCpp = db.measures().insertMetric(m -> m.setKey(UNANALYZED_CPP_KEY));
353     ComponentDto project1 = db.components().insertPublicProject();
354     ComponentDto project2 = db.components().insertPublicProject();
355     db.measures().insertLiveMeasure(project1, unanalyzedC);
356     db.measures().insertLiveMeasure(project2, unanalyzedCpp);
357
358     TelemetryData data = communityUnderTest.load();
359
360     assertThat(data.getProjectStatistics())
361       .extracting(TelemetryData.ProjectStatistics::hasUnanalyzedC, TelemetryData.ProjectStatistics::hasUnanalyzedCpp)
362       .containsExactlyInAnyOrder(tuple(Optional.empty(), Optional.empty()), tuple(Optional.empty(), Optional.empty()));
363   }
364
365   @Test
366   public void populate_security_custom_config_for_languages_on_enterprise() {
367     when(editionProvider.get()).thenReturn(Optional.of(ENTERPRISE));
368
369     when(configuration.get("sonar.security.config.javasecurity")).thenReturn(Optional.of("{}"));
370     when(configuration.get("sonar.security.config.phpsecurity")).thenReturn(Optional.of("{}"));
371     when(configuration.get("sonar.security.config.pythonsecurity")).thenReturn(Optional.of("{}"));
372     when(configuration.get("sonar.security.config.roslyn.sonaranalyzer.security.cs")).thenReturn(Optional.of("{}"));
373
374     TelemetryData data = commercialUnderTest.load();
375
376     assertThat(data.getCustomSecurityConfigs())
377       .containsExactlyInAnyOrder("java", "php", "python", "csharp");
378   }
379
380   @Test
381   public void skip_security_custom_config_on_community() {
382     when(editionProvider.get()).thenReturn(Optional.of(COMMUNITY));
383
384     TelemetryData data = communityUnderTest.load();
385
386     assertThat(data.getCustomSecurityConfigs()).isEmpty();
387   }
388
389   @Test
390   public void undetected_alm_ci_slm_data() {
391     server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4");
392     db.components().insertPublicProject();
393     TelemetryData data = communityUnderTest.load();
394     assertThat(data.getProjectStatistics())
395       .extracting(TelemetryData.ProjectStatistics::getDevopsPlatform, TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi)
396       .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected"));
397   }
398
399   @Test
400   @UseDataProvider("getScimFeatureStatues")
401   public void detect_scim_feature_status(boolean isEnabled) {
402     db.components().insertPublicProject();
403     when(configuration.getBoolean(SCIM_PROPERTY_ENABLED)).thenReturn(Optional.of(isEnabled));
404
405     TelemetryData data = communityUnderTest.load();
406
407     assertThat(data.isScimEnabled()).isEqualTo(isEnabled);
408   }
409
410   private PluginInfo newPlugin(String key, String version) {
411     return new PluginInfo(key)
412       .setVersion(Version.create(version));
413   }
414
415   private void insertAnalysisProperty(SnapshotDto snapshotDto, String uuid, String key, String value) {
416     db.getDbClient().analysisPropertiesDao().insert(db.getSession(), new AnalysisPropertyDto()
417       .setUuid(uuid)
418       .setAnalysisUuid(snapshotDto.getUuid())
419       .setKey(key)
420       .setValue(value)
421       .setCreatedAt(1L));
422   }
423
424   @DataProvider
425   public static Set<Boolean> getScimFeatureStatues() {
426     return Set.of(true, false);
427   }
428 }