From 1c7171d0aec77e28404716cc29643db085139fc7 Mon Sep 17 00:00:00 2001 From: Klaudio Sinani Date: Tue, 29 Nov 2022 17:46:37 +0100 Subject: [PATCH] SONAR-17645 Include SCIM feature status in telemetry data --- .../sonar/server/telemetry/TelemetryData.java | 38 +++++++---- .../telemetry/TelemetryDataJsonWriter.java | 9 ++- .../TelemetryDataJsonWriterTest.java | 30 +++++++-- .../telemetry/TelemetryDataLoaderImpl.java | 65 +++++++++++++------ .../TelemetryDataLoaderImplTest.java | 41 ++++++++++-- 5 files changed, 140 insertions(+), 43 deletions(-) diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java index d8caa44576c..c2825545c8a 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java @@ -19,17 +19,19 @@ */ package org.sonar.server.telemetry; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.sonar.core.platform.EditionProvider; import org.sonar.core.platform.EditionProvider.Edition; import org.sonar.db.user.UserTelemetryDto; -import static java.util.Collections.emptyList; -import static java.util.Objects.requireNonNull; +import static java.util.Objects.requireNonNullElse; public class TelemetryData { private final String serverId; @@ -41,10 +43,11 @@ public class TelemetryData { private final Long installationDate; private final String installationVersion; private final boolean inDocker; - private final List customSecurityConfigs; + private final boolean isScimEnabled; private final List users; private final List projects; private final List projectStatistics; + private final Set customSecurityConfigs; private TelemetryData(Builder builder) { serverId = builder.serverId; @@ -56,10 +59,11 @@ public class TelemetryData { installationDate = builder.installationDate; installationVersion = builder.installationVersion; inDocker = builder.inDocker; - customSecurityConfigs = builder.customSecurityConfigs == null ? emptyList() : builder.customSecurityConfigs; + isScimEnabled = builder.isScimEnabled; users = builder.users; projects = builder.projects; projectStatistics = builder.projectStatistics; + customSecurityConfigs = requireNonNullElse(builder.customSecurityConfigs, Set.of()); } public String getServerId() { @@ -98,7 +102,11 @@ public class TelemetryData { return inDocker; } - public List getCustomSecurityConfigs() { + public boolean isScimEnabled() { + return isScimEnabled; + } + + public Set getCustomSecurityConfigs() { return customSecurityConfigs; } @@ -128,7 +136,8 @@ public class TelemetryData { private Long installationDate; private String installationVersion; private boolean inDocker = false; - private List customSecurityConfigs; + private boolean isScimEnabled; + private Set customSecurityConfigs; private List users; private List projects; private List projectStatistics; @@ -182,7 +191,7 @@ public class TelemetryData { return this; } - Builder setCustomSecurityConfigs(List customSecurityConfigs) { + Builder setCustomSecurityConfigs(Set customSecurityConfigs) { this.customSecurityConfigs = customSecurityConfigs; return this; } @@ -197,12 +206,13 @@ public class TelemetryData { return this; } - TelemetryData build() { - requireNonNull(serverId); - requireNonNull(version); - requireNonNull(plugins); - requireNonNull(database); + public Builder setIsScimEnabled(boolean isEnabled) { + this.isScimEnabled = isEnabled; + return this; + } + TelemetryData build() { + requireNonNullValues(serverId, version, plugins, database); return new TelemetryData(this); } @@ -210,6 +220,10 @@ public class TelemetryData { this.projectStatistics = projectStatistics; return this; } + + private static void requireNonNullValues(Object... values) { + Arrays.stream(values).forEach(Objects::requireNonNull); + } } static class Database { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java index 8af18b0e16e..de904d212e7 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java @@ -19,6 +19,7 @@ */ package org.sonar.server.telemetry; +import com.google.common.annotations.VisibleForTesting; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -31,7 +32,9 @@ import static org.sonar.api.utils.DateUtils.DATETIME_FORMAT; public class TelemetryDataJsonWriter { - public static final String LANGUAGE_PROP = "language"; + @VisibleForTesting + static final String SCIM_PROPERTY = "scim"; + private static final String LANGUAGE_PROPERTY = "language"; public void writeTelemetryData(JsonWriter json, TelemetryData statistics) { json.beginObject(); @@ -69,6 +72,8 @@ public class TelemetryDataJsonWriter { } json.prop("docker", statistics.isInDocker()); + json.prop(SCIM_PROPERTY, statistics.isScimEnabled()); + writeUserData(json, statistics); writeProjectData(json, statistics); writeProjectStatsData(json, statistics); @@ -109,7 +114,7 @@ public class TelemetryDataJsonWriter { if (project.getLastAnalysis() != null) { json.prop("lastAnalysis", toUtc(project.getLastAnalysis())); } - json.prop(LANGUAGE_PROP, project.getLanguage()); + json.prop(LANGUAGE_PROPERTY, project.getLanguage()); json.prop("loc", project.getLoc()); json.endObject(); }); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java index b777f7f5cc8..b32a7aeb3b0 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.commons.codec.digest.DigestUtils; @@ -40,9 +41,11 @@ import org.sonar.core.platform.EditionProvider; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.user.UserTelemetryDto; +import static java.lang.String.format; import static java.util.stream.Collectors.joining; import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.server.telemetry.TelemetryDataJsonWriter.SCIM_PROPERTY; import static org.sonar.test.JsonAssert.assertJson; @RunWith(DataProviderRunner.class) @@ -210,23 +213,35 @@ public class TelemetryDataJsonWriterTest { } @Test - public void write_docker_flag() { - boolean inDocker = random.nextBoolean(); + @UseDataProvider("getFeatureFlagEnabledStates") + public void write_docker_flag(boolean isInDocker) { TelemetryData data = telemetryBuilder() - .setInDocker(inDocker) + .setInDocker(isInDocker) .build(); String json = writeTelemetryData(data); assertJson(json).isSimilarTo("{" + - " \"docker\":" + inDocker + + " \"docker\":" + isInDocker + "}"); } + @Test + @UseDataProvider("getFeatureFlagEnabledStates") + public void write_scim_feature_flag(boolean isScimEnabled) { + TelemetryData data = telemetryBuilder() + .setIsScimEnabled(isScimEnabled) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo("{" + format(" \"%s\":", SCIM_PROPERTY) + isScimEnabled + "}"); + } + @Test public void writes_security_custom_config() { TelemetryData data = telemetryBuilder() - .setCustomSecurityConfigs(Arrays.asList("php", "java")) + .setCustomSecurityConfigs(Set.of("php", "java")) .build(); String json = writeTelemetryData(data); @@ -396,4 +411,9 @@ public class TelemetryDataJsonWriterTest { } return jsonString.toString(); } + + @DataProvider + public static Set getFeatureFlagEnabledStates() { + return Set.of(true, false); + } } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java index 29b97fd2045..21f524078b1 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java @@ -19,13 +19,14 @@ */ package org.sonar.server.telemetry; +import com.google.common.annotations.VisibleForTesting; import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.CheckForNull; @@ -61,7 +62,16 @@ import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE; @ServerSide public class TelemetryDataLoaderImpl implements TelemetryDataLoader { - public static final String UNDETECTED = "undetected"; + @VisibleForTesting + static final String SCIM_PROPERTY_ENABLED = "sonar.scim.enabled"; + private static final String UNDETECTED = "undetected"; + + private static final Map LANGUAGES_BY_SECURITY_JSON_PROPERTY_MAP = Map.of( + "sonar.security.config.javasecurity", "java", + "sonar.security.config.phpsecurity", "php", + "sonar.security.config.pythonsecurity", "python", + "sonar.security.config.roslyn.sonaranalyzer.security.cs", "csharp"); + private final Server server; private final DbClient dbClient; private final PluginRepository pluginRepository; @@ -72,6 +82,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { @CheckForNull private final LicenseReader licenseReader; + @Inject public TelemetryDataLoaderImpl(Server server, DbClient dbClient, PluginRepository pluginRepository, PlatformEditionProvider editionProvider, InternalProperties internalProperties, Configuration configuration, @@ -121,9 +132,12 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { Optional installationDateProperty = internalProperties.read(InternalProperties.INSTALLATION_DATE); installationDateProperty.ifPresent(s -> data.setInstallationDate(Long.valueOf(s))); Optional installationVersionProperty = internalProperties.read(InternalProperties.INSTALLATION_VERSION); - data.setInstallationVersion(installationVersionProperty.orElse(null)); - data.setInDocker(dockerSupport.isRunningInDocker()); - return data.build(); + + return data + .setInstallationVersion(installationVersionProperty.orElse(null)) + .setInDocker(dockerSupport.isRunningInDocker()) + .setIsScimEnabled(isScimEnabled()) + .build(); } private void resolveProjectStatistics(TelemetryData.Builder data, DbSession dbSession) { @@ -185,18 +199,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { private void setSecurityCustomConfigIfPresent(TelemetryData.Builder data) { editionProvider.get() .filter(edition -> asList(ENTERPRISE, DATACENTER).contains(edition)) - .ifPresent(edition -> { - List customSecurityConfigs = new LinkedList<>(); - configuration.get("sonar.security.config.javasecurity") - .ifPresent(s -> customSecurityConfigs.add("java")); - configuration.get("sonar.security.config.phpsecurity") - .ifPresent(s -> customSecurityConfigs.add("php")); - configuration.get("sonar.security.config.pythonsecurity") - .ifPresent(s -> customSecurityConfigs.add("python")); - configuration.get("sonar.security.config.roslyn.sonaranalyzer.security.cs") - .ifPresent(s -> customSecurityConfigs.add("csharp")); - data.setCustomSecurityConfigs(customSecurityConfigs); - }); + .ifPresent(edition -> data.setCustomSecurityConfigs(getCustomerSecurityConfigurations())); } private Map getAnalysisPropertyByProject(DbSession dbSession, String analysisPropertyKey) { @@ -214,13 +217,20 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { private static String getAlmName(String alm, String url) { if (checkIfCloudAlm(alm, ALM.GITHUB.getId(), url, "https://api.github.com")) { return "github_cloud"; - } else if (checkIfCloudAlm(alm, ALM.GITLAB.getId(), url, "https://gitlab.com/api/v4")) { + } + + if (checkIfCloudAlm(alm, ALM.GITLAB.getId(), url, "https://gitlab.com/api/v4")) { return "gitlab_cloud"; - } else if (checkIfCloudAlm(alm, ALM.AZURE_DEVOPS.getId(), url, "https://dev.azure.com")) { + } + + if (checkIfCloudAlm(alm, ALM.AZURE_DEVOPS.getId(), url, "https://dev.azure.com")) { return "azure_devops_cloud"; - } else if (ALM.BITBUCKET_CLOUD.getId().equals(alm)) { + } + + if (ALM.BITBUCKET_CLOUD.getId().equals(alm)) { return alm; } + return alm + "_server"; } @@ -232,4 +242,19 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { public String loadServerId() { return server.getId(); } + + private Set getCustomerSecurityConfigurations() { + return LANGUAGES_BY_SECURITY_JSON_PROPERTY_MAP.keySet().stream() + .filter(this::isPropertyPresentInConfiguration) + .map(LANGUAGES_BY_SECURITY_JSON_PROPERTY_MAP::get) + .collect(Collectors.toSet()); + } + + private boolean isPropertyPresentInConfiguration(String property) { + return configuration.get(property).isPresent(); + } + + private boolean isScimEnabled() { + return this.configuration.getBoolean(SCIM_PROPERTY_ENABLED).orElse(false); + } } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java index 6af70b31a0b..1e384cdeba6 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java @@ -19,14 +19,21 @@ */ package org.sonar.server.telemetry; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; import org.sonar.api.config.Configuration; import org.sonar.api.impl.utils.TestSystem2; import org.sonar.core.platform.PlatformEditionProvider; @@ -39,6 +46,7 @@ import org.sonar.db.component.AnalysisPropertyDto; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.SnapshotDto; import org.sonar.db.metric.MetricDto; +import org.sonar.db.user.UserDbTester; import org.sonar.db.user.UserDto; import org.sonar.db.user.UserTelemetryDto; import org.sonar.server.platform.DockerSupport; @@ -67,7 +75,9 @@ import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE; import static org.sonar.db.component.BranchType.BRANCH; import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY; import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY; +import static org.sonar.server.telemetry.TelemetryDataLoaderImpl.SCIM_PROPERTY_ENABLED; +@RunWith(DataProviderRunner.class) public class TelemetryDataLoaderImplTest { private final static Long NOW = 100_000_000L; private final TestSystem2 system2 = new TestSystem2().setNow(NOW); @@ -101,10 +111,7 @@ public class TelemetryDataLoaderImplTest { when(pluginRepository.getPluginInfos()).thenReturn(plugins); when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER)); - int activeUserCount = 3; - List activeUsers = IntStream.range(0, activeUserCount).mapToObj(i -> db.users().insertUser( - u -> u.setExternalIdentityProvider("provider" + i).setLastSonarlintConnectionDate(i * 2L))) - .collect(Collectors.toList()); + List activeUsers = composeActiveUsers(3); // update last connection activeUsers.forEach(u -> db.users().updateLastConnectionDate(u, 5L)); @@ -178,6 +185,17 @@ public class TelemetryDataLoaderImplTest { tuple(1L, 0L, "scm-2", "ci-2", "github_cloud")); } + private List composeActiveUsers(int count) { + UserDbTester userDbTester = db.users(); + Function> userConfigurator = index -> user -> user.setExternalIdentityProvider("provider" + index).setLastSonarlintConnectionDate(index * 2L); + + return IntStream + .rangeClosed(1, count) + .mapToObj(userConfigurator::apply) + .map(userDbTester::insertUser) + .collect(Collectors.toList()); + } + private void assertDatabaseMetadata(TelemetryData.Database database) { try (DbSession dbSession = db.getDbClient().openSession(false)) { DatabaseMetaData metadata = dbSession.getConnection().getMetaData(); @@ -378,6 +396,17 @@ public class TelemetryDataLoaderImplTest { .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected")); } + @Test + @UseDataProvider("getScimFeatureStatues") + public void detect_scim_feature_status(boolean isEnabled) { + db.components().insertPublicProject(); + when(configuration.getBoolean(SCIM_PROPERTY_ENABLED)).thenReturn(Optional.of(isEnabled)); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.isScimEnabled()).isEqualTo(isEnabled); + } + private PluginInfo newPlugin(String key, String version) { return new PluginInfo(key) .setVersion(Version.create(version)); @@ -392,4 +421,8 @@ public class TelemetryDataLoaderImplTest { .setCreatedAt(1L)); } + @DataProvider + public static Set getScimFeatureStatues() { + return Set.of(true, false); + } } -- 2.39.5