From 39a12face0308a8cef9a2a3c16dd2012842ccff7 Mon Sep 17 00:00:00 2001 From: Eric Giffon Date: Fri, 2 Jun 2023 11:21:52 +0200 Subject: [PATCH] SONAR-19425 Update telemetry with kubernetes usage --- .../sonar/server/telemetry/TelemetryData.java | 3 +- .../telemetry/TelemetryDataJsonWriter.java | 3 + .../TelemetryDataJsonWriterTest.java | 19 +- .../telemetry/CloudUsageDataProvider.java | 177 +++++++++++++++++- .../telemetry/CloudUsageDataProviderTest.java | 132 ++++++++++++- .../TelemetryDataLoaderImplTest.java | 9 + .../org/sonar/server/telemetry/dummy.crt | 15 ++ 7 files changed, 351 insertions(+), 7 deletions(-) create mode 100644 server/sonar-webserver-core/src/test/resources/org/sonar/server/telemetry/dummy.crt diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java index 2f980f50d57..44a1a473096 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java @@ -352,7 +352,8 @@ public class TelemetryData { record ManagedInstanceInformation(boolean isManaged, @Nullable String provider) { } - record CloudUsage(boolean kubernetes) { + record CloudUsage(boolean kubernetes, @Nullable String kubernetesVersion, @Nullable String kubernetesPlatform, + @Nullable String kubernetesProvider) { } public static class ProjectStatistics { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java index d5243c541f7..f802d1ba1a8 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java @@ -236,6 +236,9 @@ public class TelemetryDataJsonWriter { json.name(CLOUD_USAGE_PROPERTY); json.beginObject(); json.prop("kubernetes", cloudUsage.kubernetes()); + json.prop("kubernetesVersion", cloudUsage.kubernetesVersion()); + json.prop("kubernetesPlatform", cloudUsage.kubernetesPlatform()); + json.prop("kubernetesProvider", cloudUsage.kubernetesProvider()); json.endObject(); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java index dcec4033191..2d035ca8918 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java @@ -279,6 +279,23 @@ public class TelemetryDataJsonWriterTest { } } + @Test + public void writeTelemetryData_shouldWriteCloudUsage() { + TelemetryData data = telemetryBuilder().build(); + + String json = writeTelemetryData(data); + assertJson(json).isSimilarTo(""" + { + "cloudUsage": { + "kubernetes": true, + "kubernetesVersion": "1.27", + "kubernetesPlatform": "linux/amd64", + "kubernetesProvider": "5.4.181-99.354.amzn2.x86_64" + } + } + """); + } + @Test public void writes_has_unanalyzed_languages() { TelemetryData data = telemetryBuilder() @@ -585,7 +602,7 @@ public class TelemetryDataJsonWriterTest { .setMessageSequenceNumber(1L) .setPlugins(Collections.emptyMap()) .setManagedInstanceInformation(new TelemetryData.ManagedInstanceInformation(false, null)) - .setCloudUsage(new TelemetryData.CloudUsage(false)) + .setCloudUsage(new TelemetryData.CloudUsage(true, "1.27", "linux/amd64", "5.4.181-99.354.amzn2.x86_64")) .setDatabase(new TelemetryData.Database("H2", "11")) .setNcdId(NCD_ID); } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/CloudUsageDataProvider.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/CloudUsageDataProvider.java index 67bf2e1cd9a..6fe36fcbbcd 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/CloudUsageDataProvider.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/CloudUsageDataProvider.java @@ -19,16 +19,187 @@ */ package org.sonar.server.telemetry; +import com.google.common.annotations.VisibleForTesting; +import com.google.gson.Gson; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.SecureRandom; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.util.Collection; +import java.util.Scanner; +import javax.annotation.CheckForNull; +import javax.inject.Inject; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okhttp3.internal.tls.OkHostnameVerifier; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.server.ServerSide; +import org.sonar.api.utils.System2; +import org.sonar.server.util.Paths2; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; @ServerSide public class CloudUsageDataProvider { + private static final Logger LOG = LoggerFactory.getLogger(CloudUsageDataProvider.class); + + private static final String SERVICEACCOUNT_CA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; + static final String KUBERNETES_SERVICE_HOST = "KUBERNETES_SERVICE_HOST"; + static final String KUBERNETES_SERVICE_PORT = "KUBERNETES_SERVICE_PORT"; + private static final String[] KUBERNETES_PROVIDER_COMMAND = {"bash", "-c", "uname -r"}; + private final System2 system2; + private final Paths2 paths2; + private OkHttpClient httpClient; + + @Inject + public CloudUsageDataProvider(System2 system2, Paths2 paths2) { + this.system2 = system2; + this.paths2 = paths2; + if (isOnKubernetes()) { + initHttpClient(); + } + } + + @VisibleForTesting + CloudUsageDataProvider(System2 system2, Paths2 paths2, OkHttpClient httpClient) { + this.system2 = system2; + this.paths2 = paths2; + this.httpClient = httpClient; + } + public TelemetryData.CloudUsage getCloudUsage() { - return new TelemetryData.CloudUsage(isKubernetes()); + String kubernetesVersion = null; + String kubernetesPlatform = null; + + if (isOnKubernetes()) { + VersionInfo versionInfo = getVersionInfo(); + if (versionInfo != null) { + kubernetesVersion = versionInfo.major() + "." + versionInfo.minor(); + kubernetesPlatform = versionInfo.platform(); + } + } + + return new TelemetryData.CloudUsage( + isOnKubernetes(), + kubernetesVersion, + kubernetesPlatform, + getKubernetesProvider()); + } + + private boolean isOnKubernetes() { + return StringUtils.isNotBlank(system2.envVariable(KUBERNETES_SERVICE_HOST)); + } + + /** + * Create a http client to call the Kubernetes API. + * This is based on the client creation in the official Kubernetes Java client. + */ + private void initHttpClient() { + try { + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(getKeyStore()); + TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagers, new SecureRandom()); + + httpClient = new OkHttpClient.Builder() + .sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]) + .hostnameVerifier(OkHostnameVerifier.INSTANCE) + .build(); + } catch (Exception e) { + LOG.debug("Failed to create http client for Kubernetes API", e); + } + } + + private KeyStore getKeyStore() throws GeneralSecurityException, IOException { + KeyStore caKeyStore = newEmptyKeyStore(); + + try (FileInputStream fis = new FileInputStream(paths2.get(SERVICEACCOUNT_CA_PATH).toFile())) { + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + Collection certificates = certificateFactory.generateCertificates(fis); + + int index = 0; + for (Certificate certificate : certificates) { + String certificateAlias = "ca" + index; + caKeyStore.setCertificateEntry(certificateAlias, certificate); + index++; + } + } + + return caKeyStore; + } + + private static KeyStore newEmptyKeyStore() throws GeneralSecurityException, IOException { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null, null); + return keyStore; + } + + record VersionInfo(String major, String minor, String platform) { + } + + private VersionInfo getVersionInfo() { + try { + Request request = buildRequest(); + try (Response response = httpClient.newCall(request).execute()) { + ResponseBody responseBody = requireNonNull(response.body(), "Response body is null"); + return new Gson().fromJson(responseBody.string(), VersionInfo.class); + } + } catch (Exception e) { + LOG.debug("Failed to get Kubernetes version info", e); + return null; + } + } + + private Request buildRequest() throws URISyntaxException { + String host = system2.envVariable(KUBERNETES_SERVICE_HOST); + String port = system2.envVariable(KUBERNETES_SERVICE_PORT); + if (host == null || port == null) { + throw new IllegalStateException("Kubernetes environment variables are not set"); + } + + URI uri = new URI("https", null, host, Integer.parseInt(port), "/version", null, null); + + return new Request.Builder() + .get() + .url(uri.toString()) + .build(); + } + + @CheckForNull + private static String getKubernetesProvider() { + try { + Process process = new ProcessBuilder().command(KUBERNETES_PROVIDER_COMMAND).start(); + try (Scanner scanner = new Scanner(process.getInputStream(), UTF_8)) { + scanner.useDelimiter("\n"); + return scanner.next(); + } finally { + process.destroy(); + } + } catch (Exception e) { + LOG.debug("Failed to get Kubernetes provider", e); + return null; + } } - private static boolean isKubernetes() { - return true; + @VisibleForTesting + OkHttpClient getHttpClient() { + return httpClient; } } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/CloudUsageDataProviderTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/CloudUsageDataProviderTest.java index a7ea23037d8..cb6aba86d97 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/CloudUsageDataProviderTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/CloudUsageDataProviderTest.java @@ -19,16 +19,144 @@ */ package org.sonar.server.telemetry; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Paths; +import javax.annotation.Nullable; +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import org.sonar.api.utils.System2; +import org.sonar.server.util.Paths2; +import org.sonarqube.ws.MediaTypes; +import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.server.telemetry.CloudUsageDataProvider.KUBERNETES_SERVICE_HOST; +import static org.sonar.server.telemetry.CloudUsageDataProvider.KUBERNETES_SERVICE_PORT; public class CloudUsageDataProviderTest { - private final CloudUsageDataProvider underTest = new CloudUsageDataProvider(); + private final System2 system2 = Mockito.mock(System2.class); + private final Paths2 paths2 = Mockito.mock(Paths2.class); + private final OkHttpClient httpClient = Mockito.mock(OkHttpClient.class); + private final CloudUsageDataProvider underTest = new CloudUsageDataProvider(system2, paths2, httpClient); + + @Before + public void setUp() throws Exception { + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn("localhost"); + when(system2.envVariable(KUBERNETES_SERVICE_PORT)).thenReturn("443"); + + mockHttpClientCall(200, "OK", ResponseBody.create(""" + { + "major": "1", + "minor": "25", + "gitVersion": "v1.25.3", + "gitCommit": "434bfd82814af038ad94d62ebe59b133fcb50506", + "gitTreeState": "clean", + "buildDate": "2022-11-02T03:24:50Z", + "goVersion": "go1.19.2", + "compiler": "gc", + "platform": "linux/arm64" + } + """, MediaType.parse(MediaTypes.JSON))); + } + + private void mockHttpClientCall(int code, String message, @Nullable ResponseBody body) throws IOException { + Call callMock = mock(Call.class); + when(callMock.execute()).thenReturn(new Response.Builder() + .request(new Request.Builder().url("http://any.test/").build()) + .protocol(Protocol.HTTP_1_1) + .code(code) + .message(message) + .body(body) + .build()); + when(httpClient.newCall(any())).thenReturn(callMock); + } @Test - public void isKubernetes_shouldReturnValue() { + public void kubernetes_whenEnvVarExists_shouldReturnTrue() { assertThat(underTest.getCloudUsage().kubernetes()).isTrue(); } + + @Test + public void kubernetes_whenEnvVarDoesNotExist_shouldReturnFalse() { + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn(null); + assertThat(underTest.getCloudUsage().kubernetes()).isFalse(); + } + + @Test + public void kubernetesVersion_whenOnKubernetes_shouldReturnValue() { + assertThat(underTest.getCloudUsage().kubernetesVersion()).isEqualTo("1.25"); + } + + @Test + public void kubernetesVersion_whenNotOnKubernetes_shouldReturnNull() { + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn(null); + assertThat(underTest.getCloudUsage().kubernetesVersion()).isNull(); + } + + @Test + public void kubernetesVersion_whenApiCallFails_shouldReturnNull() throws IOException { + mockHttpClientCall(404, "not found", null); + assertThat(underTest.getCloudUsage().kubernetesVersion()).isNull(); + } + + @Test + public void kubernetesPlatform_whenOnKubernetes_shouldReturnValue() { + assertThat(underTest.getCloudUsage().kubernetesPlatform()).isEqualTo("linux/arm64"); + } + + @Test + public void kubernetesPlatform_whenNotOnKubernetes_shouldReturnNull() { + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn(null); + assertThat(underTest.getCloudUsage().kubernetesPlatform()).isNull(); + } + + @Test + public void kubernetesPlatform_whenApiCallFails_shouldReturnNull() throws IOException { + mockHttpClientCall(404, "not found", null); + assertThat(underTest.getCloudUsage().kubernetesPlatform()).isNull(); + } + + @Test + public void kubernetesProvider_shouldReturnValue() { + assertThat(underTest.getCloudUsage().kubernetesProvider()).isNotBlank(); + } + + @Test + public void initHttpClient_whenValidCertificate_shouldCreateClient() throws URISyntaxException { + when(paths2.get(anyString())).thenReturn(Paths.get(requireNonNull(getClass().getResource("dummy.crt")).toURI())); + + CloudUsageDataProvider provider = new CloudUsageDataProvider(system2, paths2); + assertThat(provider.getHttpClient()).isNotNull(); + } + + @Test + public void initHttpClient_whenNotOnKubernetes_shouldNotCreateClient() throws URISyntaxException { + when(paths2.get(anyString())).thenReturn(Paths.get(requireNonNull(getClass().getResource("dummy.crt")).toURI())); + when(system2.envVariable(KUBERNETES_SERVICE_HOST)).thenReturn(null); + + CloudUsageDataProvider provider = new CloudUsageDataProvider(system2, paths2); + assertThat(provider.getHttpClient()).isNull(); + } + + @Test + public void initHttpClient_whenCertificateNotFound_shouldFail() { + when(paths2.get(any())).thenReturn(Paths.get("dummy.crt")); + + CloudUsageDataProvider provider = new CloudUsageDataProvider(system2, paths2); + assertThat(provider.getHttpClient()).isNull(); + } } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java index 2ffedd16707..ea81a4451aa 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java @@ -554,6 +554,15 @@ public class TelemetryDataLoaderImplTest { assertThat(managedInstance.provider()).isEqualTo(provider); } + @Test + public void load_shouldContainCloudUsage() { + TelemetryData.CloudUsage cloudUsage = new TelemetryData.CloudUsage(true, "1.27", "linux/amd64", "5.4.181-99.354.amzn2.x86_64"); + when(cloudUsageDataProvider.getCloudUsage()).thenReturn(cloudUsage); + + TelemetryData data = commercialUnderTest.load(); + assertThat(data.getCloudUsage()).isEqualTo(cloudUsage); + } + @Test public void default_quality_gate_sent_with_project() { db.components().insertPublicProject().getMainBranchComponent(); diff --git a/server/sonar-webserver-core/src/test/resources/org/sonar/server/telemetry/dummy.crt b/server/sonar-webserver-core/src/test/resources/org/sonar/server/telemetry/dummy.crt new file mode 100644 index 00000000000..10e8a4a760e --- /dev/null +++ b/server/sonar-webserver-core/src/test/resources/org/sonar/server/telemetry/dummy.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICQjCCAaugAwIBAgIBADANBgkqhkiG9w0BAQ0FADA9MQswCQYDVQQGEwJ1czEO +MAwGA1UECAwFZHVtbXkxDjAMBgNVBAoMBWR1bW15MQ4wDAYDVQQDDAVkdW1teTAg +Fw0yMzA2MDkxMDMxMzRaGA8yMjk3MDMyNDEwMzEzNFowPTELMAkGA1UEBhMCdXMx +DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEOMAwGA1UEAwwFZHVtbXkw +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAPL0Byqouz9UNBFRLqRRuNdGniwh +LzheMFKsdQIasTddfbsne6IuqMIBRyNr/icPrxXZEx/LY7mlKpBCYM/yty5ngEon +0QTTw/GXj3A7eDcpYD/0pVRKFcKNFIp58IKV09to2h4ttQUdjMqLS2yjc0ADugmy +ctlTR90Yna31Gi/nAgMBAAGjUDBOMB0GA1UdDgQWBBSMaHVg1zegjAH8CEZdN87I +FtN/6jAfBgNVHSMEGDAWgBSMaHVg1zegjAH8CEZdN87IFtN/6jAMBgNVHRMEBTAD +AQH/MA0GCSqGSIb3DQEBDQUAA4GBAFRViPwyPMBY6auUmaywjeLqtVPfn58MNssN +TZEh4ft3d2Z531m5thtSiZhnKFU/f1xRecUXK3jew8/RAKVSsTH7A4NYfhu5Bs/K +JfFWv7NYwL5ntnaBQZQ5uSYwPwiTYZzFrTgEDDOkXpsf7g5A16hS/L11A1lwx9b4 +WWCrjmv4 +-----END CERTIFICATE----- -- 2.39.5