From ac3afb832969385223a6c774a1bc9bb9b4f67162 Mon Sep 17 00:00:00 2001 From: Jacek Date: Fri, 29 Jan 2021 16:36:53 +0100 Subject: [PATCH] SONAR-14369 Add ALM integration telemetry --- .../sonar/server/telemetry/TelemetryData.java | 13 +++ .../telemetry/TelemetryDataJsonWriter.java | 10 +++ .../TelemetryDataJsonWriterTest.java | 28 ++++++ .../telemetry/TelemetryDataLoaderImpl.java | 26 ++++++ .../platform/ClusterSystemInfoWriterTest.java | 2 +- .../StandaloneSystemInfoWriterTest.java | 2 +- .../server/telemetry/TelemetryDaemonTest.java | 1 + .../TelemetryDataLoaderImplTest.java | 19 +++++ .../almsettings/AlmSettingsService.java | 16 ++++ .../CreateBitbucketCloudRequest.java | 85 +++++++++++++++++++ 10 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 sonar-ws/src/main/java/org/sonarqube/ws/client/almsettings/CreateBitbucketCloudRequest.java diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java index bb8ea70944e..9f18a800eba 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java @@ -37,6 +37,7 @@ public class TelemetryData { private final boolean usingBranches; private final Database database; private final Map projectCountByLanguage; + private final Map almIntegrationCountByAlm; private final Map nclocByLanguage; private final EditionProvider.Edition edition; private final String licenseType; @@ -56,6 +57,7 @@ public class TelemetryData { usingBranches = builder.usingBranches; database = builder.database; projectCountByLanguage = builder.projectMeasuresStatistics.getProjectCountByLanguage(); + almIntegrationCountByAlm = builder.almIntegrationCountByAlm; nclocByLanguage = builder.projectMeasuresStatistics.getNclocByLanguage(); edition = builder.edition; licenseType = builder.licenseType; @@ -102,6 +104,10 @@ public class TelemetryData { return projectCountByLanguage; } + public Map getAlmIntegrationCountByAlm() { + return almIntegrationCountByAlm; + } + public Map getNclocByLanguage() { return nclocByLanguage; } @@ -145,6 +151,7 @@ public class TelemetryData { private Map plugins; private Database database; private ProjectMeasuresStatistics projectMeasuresStatistics; + private Map almIntegrationCountByAlm; private Long ncloc; private Boolean usingBranches; private EditionProvider.Edition edition; @@ -179,6 +186,11 @@ public class TelemetryData { return this; } + Builder setAlmIntegrationCountByAlm(Map almIntegrationCountByAlm) { + this.almIntegrationCountByAlm = almIntegrationCountByAlm; + return this; + } + Builder setProjectMeasuresStatistics(ProjectMeasuresStatistics projectMeasuresStatistics) { this.projectMeasuresStatistics = projectMeasuresStatistics; return this; @@ -239,6 +251,7 @@ public class TelemetryData { requireNonNull(version); requireNonNull(plugins); requireNonNull(projectMeasuresStatistics); + requireNonNull(almIntegrationCountByAlm); requireNonNull(ncloc); requireNonNull(database); requireNonNull(usingBranches); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java index f70362ea48a..50fc376f6af 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java @@ -68,6 +68,16 @@ public class TelemetryDataJsonWriter { json.endObject(); }); json.endArray(); + json.name("almIntegrationCount"); + json.beginArray(); + statistics.getAlmIntegrationCountByAlm().forEach((alm, count) -> { + json.beginObject(); + json.prop("alm", alm); + json.prop("count", count); + json.endObject(); + }); + json.endArray(); + statistics.hasUnanalyzedC().ifPresent(hasUnanalyzedC -> json.prop("hasUnanalyzedC", hasUnanalyzedC)); statistics.hasUnanalyzedCpp().ifPresent(hasUnanalyzedCpp -> json.prop("hasUnanalyzedCpp", hasUnanalyzedCpp)); if (statistics.getInstallationDate() != null) { diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java index 68d26d31ff9..9c4ce3f7a38 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java @@ -19,6 +19,7 @@ */ package org.sonar.server.telemetry; +import com.google.common.collect.ImmutableMap; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; @@ -48,6 +49,7 @@ public class TelemetryDataJsonWriterTest { .setServerId("foo") .setVersion("bar") .setPlugins(Collections.emptyMap()) + .setAlmIntegrationCountByAlm(Collections.emptyMap()) .setProjectMeasuresStatistics(ProjectMeasuresStatistics.builder() .setProjectCount(12) .setProjectCountByLanguage(Collections.emptyMap()) @@ -235,6 +237,32 @@ public class TelemetryDataJsonWriterTest { "}"); } + @Test + public void write_alm_count_by_alm() { + TelemetryData data = SOME_TELEMETRY_DATA + .setAlmIntegrationCountByAlm(ImmutableMap.of( + "github", 4L, + "github_cloud", 1L, + "gitlab", 2L, + "gitlab_cloud", 5L, + "azure_devops", 1L)) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo("{" + + " \"almIntegrationCount\": " + + "[" + + "{ \"alm\":\"github\", \"count\":4}," + + "{ \"alm\":\"github_cloud\", \"count\":1}," + + "{ \"alm\":\"gitlab\", \"count\":2}," + + "{ \"alm\":\"gitlab_cloud\", \"count\":5}," + + "{ \"alm\":\"azure_devops\", \"count\":1}," + + "]" + + + "}"); + } + @Test public void does_not_write_installation_date_if_null() { TelemetryData data = SOME_TELEMETRY_DATA diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java index 56c44bf790a..93f086e52e0 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java @@ -24,6 +24,7 @@ import java.sql.SQLException; import java.util.Map; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collectors; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.sonar.api.platform.Server; @@ -34,6 +35,8 @@ import org.sonar.core.platform.PluginRepository; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.alm.setting.ALM; +import org.sonar.db.alm.setting.AlmSettingDto; import org.sonar.db.measure.SumNclocDbQuery; import org.sonar.server.es.SearchOptions; import org.sonar.server.measure.index.ProjectMeasuresIndex; @@ -45,6 +48,7 @@ import org.sonar.server.user.index.UserIndex; import org.sonar.server.user.index.UserQuery; import static java.util.Optional.ofNullable; +import static org.apache.commons.lang.StringUtils.startsWith; import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY; import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY; import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY; @@ -122,6 +126,8 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { data.setHasUnanalyzedC(numberOfUnanalyzedCMeasures > 0); data.setHasUnanalyzedCpp(numberOfUnanalyzedCppMeasures > 0); }); + + data.setAlmIntegrationCountByAlm(countAlmUsage(dbSession)); } Optional installationDateProperty = internalProperties.read(InternalProperties.INSTALLATION_DATE); installationDateProperty.ifPresent(s -> data.setInstallationDate(Long.valueOf(s))); @@ -132,6 +138,26 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { return data.build(); } + private Map countAlmUsage(DbSession dbSession) { + return dbClient.almSettingDao().selectAll(dbSession).stream() + .collect(Collectors.groupingBy(almSettingDto -> { + if (checkIfCloudAlm(almSettingDto, ALM.GITHUB, "https://api.github.com")) { + return "github_cloud"; + } else if (checkIfCloudAlm(almSettingDto, ALM.GITLAB, "https://gitlab.com/api/v4")) { + return "gitlab_cloud"; + } else if (checkIfCloudAlm(almSettingDto, ALM.AZURE_DEVOPS, "https://dev.azure.com")) { + return "azure_devops_cloud"; + } else if (ALM.BITBUCKET_CLOUD.equals(almSettingDto.getAlm())) { + return almSettingDto.getRawAlm(); + } + return almSettingDto.getRawAlm() + "_server"; + }, Collectors.counting())); + } + + private static boolean checkIfCloudAlm(AlmSettingDto almSettingDto, ALM alm, String url) { + return alm.equals(almSettingDto.getAlm()) && startsWith(almSettingDto.getUrl(), url); + } + @Override public String loadServerId() { return server.getId(); diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/ClusterSystemInfoWriterTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/ClusterSystemInfoWriterTest.java index c1e452e2f28..fe3cb49a125 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/ClusterSystemInfoWriterTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/ClusterSystemInfoWriterTest.java @@ -74,7 +74,7 @@ public class ClusterSystemInfoWriterTest { + "\"Search Nodes\":[{\"Name\":\"searchNodes\",\"\":{\"name\":\"searchNodes\"}}]," + "\"Statistics\":{\"id\":\"\",\"version\":\"\",\"database\":{\"name\":\"\",\"version\":\"\"},\"plugins\":[]," + "\"userCount\":0,\"projectCount\":0,\"usingBranches\":false,\"ncloc\":0,\"projectCountByLanguage\":[]," + - "\"nclocByLanguage\":[],\"installationDate\":0,\"installationVersion\":\"\",\"docker\":false}}"); + "\"nclocByLanguage\":[],\"almIntegrationCount\":[],\"installationDate\":0,\"installationVersion\":\"\",\"docker\":false}}"); } private static NodeInfo createNodeInfo(String name) { diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/StandaloneSystemInfoWriterTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/StandaloneSystemInfoWriterTest.java index 141e483f9d6..20f0badf332 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/StandaloneSystemInfoWriterTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/StandaloneSystemInfoWriterTest.java @@ -81,7 +81,7 @@ public class StandaloneSystemInfoWriterTest { // response does not contain empty "Section Three" assertThat(writer.toString()).isEqualTo("{\"Health\":\"GREEN\",\"Health Causes\":[],\"Section One\":{\"foo\":\"bar\"},\"Section Two\":{\"one\":1,\"two\":2}," + "\"Statistics\":{\"id\":\"\",\"version\":\"\",\"database\":{\"name\":\"\",\"version\":\"\"},\"plugins\":[],\"userCount\":0,\"projectCount\":0,\"usingBranches\":false," + - "\"ncloc\":0,\"projectCountByLanguage\":[],\"nclocByLanguage\":[],\"installationDate\":0,\"installationVersion\":\"\",\"docker\":false}}"); + "\"ncloc\":0,\"projectCountByLanguage\":[],\"nclocByLanguage\":[],\"almIntegrationCount\":[],\"installationDate\":0,\"installationVersion\":\"\",\"docker\":false}}"); } private void logInAsSystemAdministrator() { diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java index af2267650b2..ad8c5cea40c 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java @@ -64,6 +64,7 @@ public class TelemetryDaemonTest { .setServerId("foo") .setVersion("bar") .setPlugins(Collections.emptyMap()) + .setAlmIntegrationCountByAlm(Collections.emptyMap()) .setProjectMeasuresStatistics(ProjectMeasuresStatistics.builder() .setProjectCount(12) .setProjectCountByLanguage(Collections.emptyMap()) diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java index f7edb84d02d..a4925b0babb 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java @@ -118,6 +118,16 @@ public class TelemetryDataLoaderImplTest { db.measures().insertLiveMeasure(project2, nclocDistrib, m -> m.setValue(null).setData("java=300;kotlin=2500")); projectMeasuresIndexer.indexAll(); + // alm + db.almSettings().insertAzureAlmSetting(); + db.almSettings().insertAzureAlmSetting(a -> a.setUrl("https://dev.azure.com")); + db.almSettings().insertBitbucketAlmSetting(); + db.almSettings().insertBitbucketCloudAlmSetting(); + db.almSettings().insertGitHubAlmSetting(); + db.almSettings().insertGitHubAlmSetting(a -> a.setUrl("https://api.github.com")); + db.almSettings().insertGitlabAlmSetting(); + db.almSettings().insertGitlabAlmSetting(a -> a.setUrl("https://gitlab.com/api/v4")); + TelemetryData data = communityUnderTest.load(); assertThat(data.getServerId()).isEqualTo(serverId); assertThat(data.getVersion()).isEqualTo(version); @@ -133,6 +143,15 @@ public class TelemetryDataLoaderImplTest { assertThat(data.getNclocByLanguage()).containsOnly( entry("java", 500L), entry("kotlin", 2500L), entry("js", 50L)); assertThat(data.isInDocker()).isFalse(); + assertThat(data.getAlmIntegrationCountByAlm()) + .containsEntry("azure_devops_server", 1L) + .containsEntry("azure_devops_cloud", 1L) + .containsEntry("bitbucket_server", 1L) + .containsEntry("bitbucket_cloud", 1L) + .containsEntry("gitlab_server", 1L) + .containsEntry("gitlab_cloud", 1L) + .containsEntry("github_cloud", 1L) + .containsEntry("github_server", 1L); } private void assertDatabaseMetadata(TelemetryData.Database database) { diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/almsettings/AlmSettingsService.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/almsettings/AlmSettingsService.java index db2822356c3..f109c608153 100644 --- a/sonar-ws/src/main/java/org/sonarqube/ws/client/almsettings/AlmSettingsService.java +++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/almsettings/AlmSettingsService.java @@ -95,6 +95,22 @@ public class AlmSettingsService extends BaseService { .setMediaType(MediaTypes.JSON)).content(); } + /** + * + * This is a POST request. + * @see Further information about this action online (including a response example) + * @since 8.7 + */ + public void createBitbucketCloud(CreateBitbucketCloudRequest request) { + call( + new PostRequest(path("create_bitbucketcloud")) + .setParam("key", request.getKey()) + .setParam("clientId", request.getClientId()) + .setParam("clientSecret", request.getClientSecret()) + .setParam("workspace", request.getWorkspace()) + .setMediaType(MediaTypes.JSON)).content(); + } + /** * This is a POST request. * @see Further information about this action online (including a response example) diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/almsettings/CreateBitbucketCloudRequest.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/almsettings/CreateBitbucketCloudRequest.java new file mode 100644 index 00000000000..92e2fb255a3 --- /dev/null +++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/almsettings/CreateBitbucketCloudRequest.java @@ -0,0 +1,85 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.ws.client.almsettings; + +import javax.annotation.Generated; + +/** + * + * This is a POST request. + * @see Further information about this action online (including a response example) + * @since 8.7 + */ +@Generated("sonar-ws-generator") +public class CreateBitbucketCloudRequest { + + private String key; + private String clientId; + private String clientSecret; + private String workspace; + + public String getKey() { + return key; + } + + /** + * This is a mandatory parameter. + */ + public CreateBitbucketCloudRequest setKey(String key) { + this.key = key; + return this; + } + + public String getClientId() { + return clientId; + } + + /** + * This is a mandatory parameter. + */ + public CreateBitbucketCloudRequest setClientId(String clientId) { + this.clientId = clientId; + return this; + } + + public String getClientSecret() { + return clientSecret; + } + + /** + * This is a mandatory parameter. + */ + public CreateBitbucketCloudRequest setClientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public String getWorkspace() { + return workspace; + } + + /** + * This is a mandatory parameter. + */ + public CreateBitbucketCloudRequest setWorkspace(String workspace) { + this.workspace = workspace; + return this; + } +} -- 2.39.5