From 0ce62a09539920b477873c2c26e9a6d2fc62a15e Mon Sep 17 00:00:00 2001 From: Alain Kermis Date: Thu, 18 Jul 2024 10:46:37 +0200 Subject: [PATCH] SONAR-22479 Introduce ADHOC granularity --- .../telemetry/TelemetryMetricsSentDaoIT.java | 8 +- .../db/telemetry/TelemetryMetricsSentDao.java | 8 +- .../org/sonar/telemetry/core/Granularity.java | 1 + .../telemetry/core/TelemetryDataProvider.java | 3 +- .../sonar/telemetry/core/GranularityTest.java | 1 + .../metrics/TelemetryMetricsLoaderIT.java | 228 +++++++++++++++++- .../org/sonar/telemetry/TelemetryDaemon.java | 2 +- .../metrics/TelemetryMetricsLoader.java | 10 +- .../metrics/TelemetryMetricsMapper.java | 9 +- .../metrics/schema/InstallationMetric.java | 3 +- .../metrics/util/SentMetricsStorage.java | 21 +- .../metrics/TelemetryMetricsMapperTest.java | 24 ++ .../metrics/TestTelemetryAdhocBean.java | 67 +++++ .../telemetry/metrics/TestTelemetryBean.java | 5 +- .../schema/InstallationMetricTest.java | 17 +- .../metrics/util/SentMetricsStorageTest.java | 24 +- .../telemetry/TelemetryVersionProvider.java | 5 +- .../TelemetryVersionProviderTest.java | 3 +- 18 files changed, 391 insertions(+), 48 deletions(-) create mode 100644 server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TestTelemetryAdhocBean.java diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/telemetry/TelemetryMetricsSentDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/telemetry/TelemetryMetricsSentDaoIT.java index cab1919d82f..d245567cb11 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/telemetry/TelemetryMetricsSentDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/telemetry/TelemetryMetricsSentDaoIT.java @@ -45,7 +45,7 @@ class TelemetryMetricsSentDaoIT { List dtos = IntStream.range(0, 10) .mapToObj(i -> TelemetryMetricsSentTesting.newTelemetryMetricsSentDto()) .toList(); - dtos.forEach(metricDto -> db.getDbClient().telemetryMetricsSentDao().insert(db.getSession(), metricDto)); + dtos.forEach(metricDto -> db.getDbClient().telemetryMetricsSentDao().upsert(db.getSession(), metricDto)); db.getSession().commit(); assertThat(underTest.selectAll(dbSession)) @@ -61,12 +61,12 @@ class TelemetryMetricsSentDaoIT { } @Test - void upsert_shouldUpdateOnly() { + void upsert_shouldUpdateOnlyAfterSecondPersistence() { TelemetryMetricsSentDto dto = TelemetryMetricsSentTesting.newTelemetryMetricsSentDto(); - underTest.insert(dbSession, dto); + underTest.upsert(dbSession, dto); system2.setNow(NOW + 1); - underTest.update(dbSession, dto); + underTest.upsert(dbSession, dto); List dtos = underTest.selectAll(dbSession); assertThat(dtos).hasSize(1); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/telemetry/TelemetryMetricsSentDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/telemetry/TelemetryMetricsSentDao.java index 5c78c7df88e..dc4d60df6a9 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/telemetry/TelemetryMetricsSentDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/telemetry/TelemetryMetricsSentDao.java @@ -36,13 +36,7 @@ public class TelemetryMetricsSentDao implements Dao { return mapper(session).selectAll(); } - public TelemetryMetricsSentDto insert(DbSession session, TelemetryMetricsSentDto dto) { - mapper(session).upsert(dto); - - return dto; - } - - public void update(DbSession dbSession, TelemetryMetricsSentDto telemetryMetricsSentDto) { + public void upsert(DbSession dbSession, TelemetryMetricsSentDto telemetryMetricsSentDto) { long now = system2.now(); telemetryMetricsSentDto.setLastSent(now); mapper(dbSession).upsert(telemetryMetricsSentDto); diff --git a/server/sonar-telemetry-core/src/main/java/org/sonar/telemetry/core/Granularity.java b/server/sonar-telemetry-core/src/main/java/org/sonar/telemetry/core/Granularity.java index be5cd22bb93..e66b2a83f87 100644 --- a/server/sonar-telemetry-core/src/main/java/org/sonar/telemetry/core/Granularity.java +++ b/server/sonar-telemetry-core/src/main/java/org/sonar/telemetry/core/Granularity.java @@ -27,6 +27,7 @@ import com.fasterxml.jackson.annotation.JsonValue; * Modifying this enum needs to be discussed beforehand with Data Platform team. */ public enum Granularity { + ADHOC("adhoc"), DAILY("daily"), WEEKLY("weekly"), MONTHLY("monthly"); diff --git a/server/sonar-telemetry-core/src/main/java/org/sonar/telemetry/core/TelemetryDataProvider.java b/server/sonar-telemetry-core/src/main/java/org/sonar/telemetry/core/TelemetryDataProvider.java index a464014e767..b29fec4057b 100644 --- a/server/sonar-telemetry-core/src/main/java/org/sonar/telemetry/core/TelemetryDataProvider.java +++ b/server/sonar-telemetry-core/src/main/java/org/sonar/telemetry/core/TelemetryDataProvider.java @@ -20,6 +20,7 @@ package org.sonar.telemetry.core; import java.util.Map; +import java.util.Optional; /** * This interface is used to provide data to the telemetry system. The telemetry system will call the methods of this interface to get the @@ -61,7 +62,7 @@ public interface TelemetryDataProvider { * * @return the value of the data provided by this instance. */ - default T getValue() { + default Optional getValue() { throw new IllegalStateException("Not implemented"); } diff --git a/server/sonar-telemetry-core/src/test/java/org/sonar/telemetry/core/GranularityTest.java b/server/sonar-telemetry-core/src/test/java/org/sonar/telemetry/core/GranularityTest.java index a4b01e93305..44ef21c9298 100644 --- a/server/sonar-telemetry-core/src/test/java/org/sonar/telemetry/core/GranularityTest.java +++ b/server/sonar-telemetry-core/src/test/java/org/sonar/telemetry/core/GranularityTest.java @@ -30,6 +30,7 @@ class GranularityTest { assertEquals("daily", Granularity.DAILY.getValue()); assertEquals("weekly", Granularity.WEEKLY.getValue()); assertEquals("monthly", Granularity.MONTHLY.getValue()); + assertEquals("adhoc", Granularity.ADHOC.getValue()); } } diff --git a/server/sonar-telemetry/src/it/java/org/sonar/telemetry/metrics/TelemetryMetricsLoaderIT.java b/server/sonar-telemetry/src/it/java/org/sonar/telemetry/metrics/TelemetryMetricsLoaderIT.java index f264fc8c107..6dc2f649365 100644 --- a/server/sonar-telemetry/src/it/java/org/sonar/telemetry/metrics/TelemetryMetricsLoaderIT.java +++ b/server/sonar-telemetry/src/it/java/org/sonar/telemetry/metrics/TelemetryMetricsLoaderIT.java @@ -19,16 +19,32 @@ */ package org.sonar.telemetry.metrics; +import com.tngtech.java.junit.dataprovider.DataProvider; import java.util.List; -import org.junit.Rule; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.sonar.api.impl.utils.TestSystem2; import org.sonar.core.util.UuidFactory; import org.sonar.db.DbTester; +import org.sonar.db.telemetry.TelemetryMetricsSentDto; import org.sonar.telemetry.FakeServer; import org.sonar.telemetry.core.Dimension; +import org.sonar.telemetry.core.Granularity; import org.sonar.telemetry.core.TelemetryDataProvider; +import org.sonar.telemetry.core.TelemetryDataType; import org.sonar.telemetry.metrics.schema.BaseMessage; +import org.sonar.telemetry.metrics.schema.InstallationMetric; +import org.sonar.telemetry.metrics.schema.LanguageMetric; +import org.sonar.telemetry.metrics.schema.ProjectMetric; +import org.sonar.telemetry.metrics.schema.UserMetric; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; @@ -40,14 +56,29 @@ class TelemetryMetricsLoaderIT { private static final String SOME_UUID = "some-uuid"; private static final Long NOW = 100_000_000L; private static final String SERVER_ID = "AU-TpxcB-iU5OvuD2FL7"; + public static final String KEY_IS_VALID = "is_valid"; + public static final String KEY_COVERAGE = "coverage"; + public static final String KEY_NCLOC = "ncloc"; + public static final String DIMENSION_INSTALLATION = "installation"; + public static final String DIMENSION_PROJECT = "project"; + public static final String DIMENSION_LANGUAGE = "language"; + public static final String KEY_LAST_ACTIVE = "last_active"; + public static final String KEY_UPDATE_FINISHED = "update_finished"; private final TestSystem2 system2 = new TestSystem2().setNow(NOW); - @Rule + @RegisterExtension public DbTester db = DbTester.create(system2); private final FakeServer server = new FakeServer(); private final UuidFactory uuidFactory = mock(UuidFactory.class); - private final List> providers = List.of(new TestTelemetryBean(Dimension.INSTALLATION), new TestTelemetryBean(Dimension.USER)); - private final TelemetryMetricsLoader underTest = new TelemetryMetricsLoader(server, db.getDbClient(), uuidFactory, providers); + private final List> providers = List.of( + getBean(KEY_IS_VALID, Dimension.INSTALLATION, Granularity.DAILY, false), + getBean(KEY_COVERAGE, Dimension.PROJECT, Granularity.WEEKLY, 12.05F, "module-1", "module-2"), + getBean(KEY_NCLOC, Dimension.LANGUAGE, Granularity.MONTHLY, 125, "java", "cpp"), + getBean(KEY_LAST_ACTIVE, Dimension.USER, Granularity.DAILY, "2024-01-01", "user-1"), + getBean(KEY_UPDATE_FINISHED, Dimension.INSTALLATION, Granularity.ADHOC, true), + getBean(KEY_UPDATE_FINISHED, Dimension.INSTALLATION, Granularity.ADHOC, false) + ); + private final TelemetryMetricsLoader underTest = new TelemetryMetricsLoader(system2, server, db.getDbClient(), uuidFactory, providers); @Test void sendTelemetryData() { @@ -55,15 +86,198 @@ class TelemetryMetricsLoaderIT { server.setId(SERVER_ID); TelemetryMetricsLoader.Context context = underTest.loadData(); + Set messages = context.getMessages(); - assertThat(context.getMessages()).hasSize(2); + assertThat(context.getMetricsToUpdate()) + .hasSize(6) + .extracting(TelemetryMetricsSentDto::getMetricKey, TelemetryMetricsSentDto::getDimension, TelemetryMetricsSentDto::getLastSent) + .containsExactlyInAnyOrder( + tuple(KEY_IS_VALID, DIMENSION_INSTALLATION, 0L), + tuple(KEY_COVERAGE, DIMENSION_PROJECT, 0L), + tuple(KEY_NCLOC, DIMENSION_LANGUAGE, 0L), + tuple(KEY_LAST_ACTIVE, "user", 0L), + tuple(KEY_UPDATE_FINISHED, DIMENSION_INSTALLATION, 0L), + tuple(KEY_UPDATE_FINISHED, DIMENSION_INSTALLATION, 0L) + ); - assertThat(context.getMessages()) + assertThat(messages) + .hasSize(4) .extracting(BaseMessage::getMessageUuid, BaseMessage::getInstallationId, BaseMessage::getDimension) .containsExactlyInAnyOrder( tuple(SOME_UUID, SERVER_ID, Dimension.INSTALLATION), - tuple(SOME_UUID, SERVER_ID, Dimension.USER) + tuple(SOME_UUID, SERVER_ID, Dimension.USER), + tuple(SOME_UUID, SERVER_ID, Dimension.PROJECT), + tuple(SOME_UUID, SERVER_ID, Dimension.LANGUAGE) + ); + + messages.forEach(message -> { + switch (message.getDimension()) { + case INSTALLATION -> assertInstallationMetrics(message); + case USER -> assertUserMetrics(message); + case LANGUAGE -> assertLanguageMetrics(message); + case PROJECT -> assertProjectMetrics(message); + default -> throw new IllegalArgumentException("Should not get here"); + } + }); + } + + @ParameterizedTest + @MethodSource("shouldNotBeUpdatedMetrics") + void loadData_whenDailyMetricsShouldNotBeSent(String key, String dimension, TimeUnit unit, int offset) { + when(uuidFactory.create()).thenReturn(SOME_UUID); + + server.setId(SERVER_ID); + + system2.setNow(1L); + TelemetryMetricsSentDto dto = new TelemetryMetricsSentDto(key, dimension); + db.getDbClient().telemetryMetricsSentDao().upsert(db.getSession(), dto); + db.commit(); + + system2.setNow(unit.toMillis(offset)); + TelemetryMetricsLoader.Context context = underTest.loadData(); + + List toUpdate = context.getMetricsToUpdate(); + + assertThat(toUpdate) + .hasSize(5) + .extracting(TelemetryMetricsSentDto::getMetricKey) + .doesNotContain(key); + } + + @ParameterizedTest + @MethodSource("shouldBeUpdatedMetrics") + void loadData_whenDailyMetricsShouldBeSent(String key, String dimension, TimeUnit unit, int offset) { + when(uuidFactory.create()).thenReturn(SOME_UUID); + + server.setId(SERVER_ID); + + system2.setNow(1L); + TelemetryMetricsSentDto dto = new TelemetryMetricsSentDto(key, dimension); + db.getDbClient().telemetryMetricsSentDao().upsert(db.getSession(), dto); + db.commit(); + + system2.setNow(unit.toMillis(offset)); + TelemetryMetricsLoader.Context context = underTest.loadData(); + + List toUpdate = context.getMetricsToUpdate(); + + assertThat(toUpdate) + .hasSize(6) + .extracting(TelemetryMetricsSentDto::getMetricKey, TelemetryMetricsSentDto::getDimension) + .contains(tuple(key, dimension)); + } + + @DataProvider + public static Object[][] shouldBeUpdatedMetrics() { + return new Object[][]{ + {KEY_IS_VALID, DIMENSION_INSTALLATION, TimeUnit.DAYS, 100}, + {KEY_COVERAGE, DIMENSION_PROJECT, TimeUnit.DAYS, 100}, + {KEY_NCLOC, DIMENSION_LANGUAGE, TimeUnit.DAYS, 100} + }; + }// 1 minute ago + + @DataProvider + public static Object[][] shouldNotBeUpdatedMetrics() { + return new Object[][]{ + {KEY_IS_VALID, DIMENSION_INSTALLATION, TimeUnit.HOURS, 1}, + {KEY_COVERAGE, DIMENSION_PROJECT, TimeUnit.DAYS, 5}, + {KEY_NCLOC, DIMENSION_LANGUAGE, TimeUnit.DAYS, 24} + }; + } + + private static void assertProjectMetrics(BaseMessage message) { + assertThat(message.getInstallationId()).isEqualTo(SERVER_ID); + assertThat(message.getDimension()).isEqualTo(Dimension.PROJECT); + assertThat((Set< ProjectMetric>) (Set) message.getMetrics()) + .extracting(ProjectMetric::getKey, ProjectMetric::getGranularity, ProjectMetric::getType, ProjectMetric::getProjectUuid, ProjectMetric::getValue) + .containsExactlyInAnyOrder( + tuple(KEY_COVERAGE, Granularity.WEEKLY, TelemetryDataType.FLOAT, "module-1", 12.05f), + tuple(KEY_COVERAGE, Granularity.WEEKLY, TelemetryDataType.FLOAT, "module-2", 12.05f) + ); + } + + private static void assertLanguageMetrics(BaseMessage message) { + assertThat(message.getInstallationId()).isEqualTo(SERVER_ID); + assertThat(message.getDimension()).isEqualTo(Dimension.LANGUAGE); + assertThat((Set< LanguageMetric>) (Set) message.getMetrics()) + .extracting(LanguageMetric::getKey, LanguageMetric::getGranularity, LanguageMetric::getType, LanguageMetric::getLanguage, LanguageMetric::getValue) + .containsExactlyInAnyOrder( + tuple(KEY_NCLOC, Granularity.MONTHLY, TelemetryDataType.INTEGER, "java", 125), + tuple(KEY_NCLOC, Granularity.MONTHLY, TelemetryDataType.INTEGER, "cpp", 125) + ); + } + + private static void assertUserMetrics(BaseMessage message) { + assertThat(message.getInstallationId()).isEqualTo(SERVER_ID); + assertThat(message.getDimension()).isEqualTo(Dimension.USER); + assertThat((Set) (Set) message.getMetrics()) + .extracting(UserMetric::getKey, UserMetric::getGranularity, UserMetric::getType, UserMetric::getUserUuid, UserMetric::getValue) + .containsExactlyInAnyOrder( + tuple(KEY_LAST_ACTIVE, Granularity.DAILY, TelemetryDataType.STRING, "user-1", "2024-01-01") ); } + private static void assertInstallationMetrics(BaseMessage message) { + assertThat(message.getInstallationId()).isEqualTo(SERVER_ID); + assertThat(message.getDimension()).isEqualTo(Dimension.INSTALLATION); + assertThat((Set) (Set) message.getMetrics()) + .extracting(InstallationMetric::getKey, InstallationMetric::getGranularity, InstallationMetric::getType, InstallationMetric::getValue) + .containsExactlyInAnyOrder( + tuple(KEY_IS_VALID, Granularity.DAILY, TelemetryDataType.BOOLEAN, false), + tuple(KEY_UPDATE_FINISHED, Granularity.ADHOC, TelemetryDataType.BOOLEAN, true) + ); + } + + private TelemetryDataProvider getBean(String key, Dimension dimension, Granularity granularity, T value, String... keys) { + return new TelemetryDataProvider<>() { + @Override + public String getMetricKey() { + return key; + } + + @Override + public Dimension getDimension() { + return dimension; + } + + @Override + public Granularity getGranularity() { + return granularity; + } + + @Override + public TelemetryDataType getType() { + if (value.getClass() == String.class) { + return TelemetryDataType.STRING; + } else if (value.getClass() == Integer.class) { + return TelemetryDataType.INTEGER; + } else if (value.getClass() == Float.class) { + return TelemetryDataType.FLOAT; + } else if (value.getClass() == Boolean.class) { + return TelemetryDataType.BOOLEAN; + } else { + throw new IllegalArgumentException("Unsupported type: " + value.getClass()); + } + } + + @Override + public Optional getValue() { + if (granularity == Granularity.ADHOC && value.getClass() == Boolean.class && !((Boolean) value)) { + return Optional.empty(); + } + + return Optional.of(value); + } + + @Override + public Map getUuidValues() { + return Stream.of(keys) + .collect(Collectors.toMap( + key -> key, + key -> value + )); + } + }; + } + } diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/TelemetryDaemon.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/TelemetryDaemon.java index 83b605e7a10..a528094695a 100644 --- a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/TelemetryDaemon.java +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/TelemetryDaemon.java @@ -166,7 +166,7 @@ public class TelemetryDaemon extends AbstractStoppableScheduledExecutorServiceIm } try (DbSession dbSession = dbClient.openSession(false)) { - context.getMetricsToUpdate().forEach(toUpdate -> dbClient.telemetryMetricsSentDao().update(dbSession, toUpdate)); + context.getMetricsToUpdate().forEach(toUpdate -> dbClient.telemetryMetricsSentDao().upsert(dbSession, toUpdate)); dbSession.commit(); } } diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/TelemetryMetricsLoader.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/TelemetryMetricsLoader.java index 5cb2e69e242..f232fe82fce 100644 --- a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/TelemetryMetricsLoader.java +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/TelemetryMetricsLoader.java @@ -28,6 +28,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.sonar.api.platform.Server; +import org.sonar.api.utils.System2; import org.sonar.core.util.UuidFactory; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -39,12 +40,15 @@ import org.sonar.telemetry.metrics.schema.Metric; import org.sonar.telemetry.metrics.util.SentMetricsStorage; public class TelemetryMetricsLoader { + private final System2 system2; private final Server server; private final DbClient dbClient; private final UuidFactory uuidFactory; private final List> providers; - public TelemetryMetricsLoader(Server server, DbClient dbClient, UuidFactory uuidFactory, List> providers) { + + public TelemetryMetricsLoader(System2 system2, Server server, DbClient dbClient, UuidFactory uuidFactory, List> providers) { + this.system2 = system2; this.server = server; this.dbClient = dbClient; this.providers = providers; @@ -63,7 +67,7 @@ public class TelemetryMetricsLoader { Map> telemetryDataMap = new LinkedHashMap<>(); for (TelemetryDataProvider provider : this.providers) { - boolean shouldSendMetric = storage.shouldSendMetric(provider.getDimension(), provider.getMetricKey(), provider.getGranularity()); + boolean shouldSendMetric = storage.shouldSendMetric(provider.getDimension(), provider.getMetricKey(), provider.getGranularity(), system2.now()); if (shouldSendMetric) { Set newMetrics = TelemetryMetricsMapper.mapFromDataProvider(provider); telemetryDataMap.computeIfAbsent(provider.getDimension(), k -> new LinkedHashSet<>()).addAll(newMetrics); @@ -88,6 +92,8 @@ public class TelemetryMetricsLoader { private Set retrieveBaseMessages(Map> metrics) { return metrics.entrySet().stream() + // we do not want to send payloads with zero metrics + .filter(v -> !v.getValue().isEmpty()) .map(entry -> new BaseMessage.Builder() .setMessageUuid(uuidFactory.create()) .setInstallationId(server.getId()) diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/TelemetryMetricsMapper.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/TelemetryMetricsMapper.java index f6ff629effb..e3379e4772f 100644 --- a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/TelemetryMetricsMapper.java +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/TelemetryMetricsMapper.java @@ -20,8 +20,10 @@ package org.sonar.telemetry.metrics; import java.util.Collections; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import org.sonar.telemetry.core.Granularity; import org.sonar.telemetry.core.TelemetryDataProvider; import org.sonar.telemetry.metrics.schema.InstallationMetric; import org.sonar.telemetry.metrics.schema.LanguageMetric; @@ -49,9 +51,14 @@ public class TelemetryMetricsMapper { } private static Set mapInstallationMetric(TelemetryDataProvider provider) { + Optional optionalValue = provider.getValue(); + if (provider.getGranularity() == Granularity.ADHOC && optionalValue.isEmpty()) { + return Collections.emptySet(); + } + return Collections.singleton(new InstallationMetric( provider.getMetricKey(), - provider.getValue(), + optionalValue.orElse(null), provider.getType(), provider.getGranularity() )); diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/schema/InstallationMetric.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/schema/InstallationMetric.java index 0f0a2abc0fe..68dc2e83590 100644 --- a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/schema/InstallationMetric.java +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/schema/InstallationMetric.java @@ -19,12 +19,13 @@ */ package org.sonar.telemetry.metrics.schema; +import javax.annotation.Nullable; import org.sonar.telemetry.core.Granularity; import org.sonar.telemetry.core.TelemetryDataType; public class InstallationMetric extends Metric { - public InstallationMetric(String key, Object value, TelemetryDataType type, Granularity granularity) { + public InstallationMetric(String key, @Nullable Object value, TelemetryDataType type, Granularity granularity) { this.key = key; this.value = value; this.type = type; diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/util/SentMetricsStorage.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/util/SentMetricsStorage.java index 3f8b7c73cc6..ca0d1d2a402 100644 --- a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/util/SentMetricsStorage.java +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/metrics/util/SentMetricsStorage.java @@ -20,8 +20,8 @@ package org.sonar.telemetry.metrics.util; import java.time.Instant; -import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.EnumMap; import java.util.HashMap; @@ -50,28 +50,29 @@ public class SentMetricsStorage { return Optional.empty(); } - public boolean shouldSendMetric(Dimension dimension, String metricKey, Granularity granularity) { + public boolean shouldSendMetric(Dimension dimension, String metricKey, Granularity granularity, long now) { + if (granularity == Granularity.ADHOC) { + return true; + } + Map metricKeyMap = dimensionMetricKeyMap.get(dimension); boolean exists = metricKeyMap != null && metricKeyMap.containsKey(metricKey); - if (!exists) { return true; } TelemetryMetricsSentDto dto = metricKeyMap.get(metricKey); - Instant lastSentTime = Instant.ofEpochMilli(dto.getLastSent()); - Instant now = Instant.now(); - LocalDateTime lastSentDateTime = LocalDateTime.ofInstant(lastSentTime, ZoneId.systemDefault()); - LocalDateTime nowDateTime = LocalDateTime.ofInstant(now, ZoneId.systemDefault()); + ZonedDateTime lastSentInstant = Instant.ofEpochMilli(dto.getLastSent()).atZone(ZoneId.systemDefault()); + ZonedDateTime nowInstant = Instant.ofEpochMilli(now).atZone(ZoneId.systemDefault()); switch (granularity) { case DAILY -> { - return ChronoUnit.DAYS.between(lastSentDateTime, nowDateTime) > 0; + return ChronoUnit.DAYS.between(lastSentInstant, nowInstant) > 0; } case WEEKLY -> { - return ChronoUnit.WEEKS.between(lastSentDateTime, nowDateTime) > 0; + return ChronoUnit.WEEKS.between(lastSentInstant, nowInstant) > 0; } case MONTHLY -> { - return ChronoUnit.MONTHS.between(lastSentDateTime, nowDateTime) > 0; + return ChronoUnit.MONTHS.between(lastSentInstant, nowInstant) > 0; } default -> throw new IllegalArgumentException("Unknown granularity: " + granularity); } } diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TelemetryMetricsMapperTest.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TelemetryMetricsMapperTest.java index a4ef44b09c3..d2fa9f9533c 100644 --- a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TelemetryMetricsMapperTest.java +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TelemetryMetricsMapperTest.java @@ -95,6 +95,30 @@ class TelemetryMetricsMapperTest { ); } + @Test + void mapFromDataProvider_whenAdhocInstallationProviderWithoutValue_shouldNotMapToMetric() { + TestTelemetryAdhocBean provider = new TestTelemetryAdhocBean(Dimension.INSTALLATION, false); // Force the value so that nothing is returned + + Set metrics = TelemetryMetricsMapper.mapFromDataProvider(provider); + List userMetrics = retrieveList(metrics); + + assertThat(userMetrics).isEmpty(); + } + + @Test + void mapFromDataProvider_whenAdhocInstallationProviderWithValue_shouldMapToMetric() { + TestTelemetryAdhocBean provider = new TestTelemetryAdhocBean(Dimension.INSTALLATION, true); // Force the value to be returned + + Set metrics = TelemetryMetricsMapper.mapFromDataProvider(provider); + List userMetrics = retrieveList(metrics); + + assertThat(userMetrics) + .extracting(InstallationMetric::getKey, InstallationMetric::getType, InstallationMetric::getValue, InstallationMetric::getGranularity) + .containsExactlyInAnyOrder( + tuple("telemetry-adhoc-bean", TelemetryDataType.BOOLEAN, true, Granularity.ADHOC) + ); + } + private static Tuple[] expected() { return new Tuple[] { diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TestTelemetryAdhocBean.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TestTelemetryAdhocBean.java new file mode 100644 index 00000000000..cedaacb40bf --- /dev/null +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TestTelemetryAdhocBean.java @@ -0,0 +1,67 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.telemetry.metrics; + +import java.util.Optional; +import org.sonar.telemetry.core.Dimension; +import org.sonar.telemetry.core.Granularity; +import org.sonar.telemetry.core.TelemetryDataProvider; +import org.sonar.telemetry.core.TelemetryDataType; + +public class TestTelemetryAdhocBean implements TelemetryDataProvider { + + private static final String METRIC_KEY = "telemetry-adhoc-bean"; + private static final Granularity METRIC_GRANULARITY = Granularity.ADHOC; + private static final TelemetryDataType METRIC_TYPE = TelemetryDataType.BOOLEAN; + + private final Dimension dimension; + private final boolean value; + + public TestTelemetryAdhocBean(Dimension dimension, boolean value) { + this.dimension = dimension; + this.value = value; + } + + @Override + public Dimension getDimension() { + return dimension; + } + + @Override + public String getMetricKey() { + return METRIC_KEY; + } + + @Override + public Granularity getGranularity() { + return METRIC_GRANULARITY; + } + + @Override + public TelemetryDataType getType() { + return METRIC_TYPE; + } + + @Override + public Optional getValue() { + return value ? Optional.of(true) : Optional.empty(); + } + +} diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TestTelemetryBean.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TestTelemetryBean.java index e19824e84b0..7aa64e464dc 100644 --- a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TestTelemetryBean.java +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/TestTelemetryBean.java @@ -20,6 +20,7 @@ package org.sonar.telemetry.metrics; import java.util.Map; +import java.util.Optional; import org.sonar.telemetry.core.Dimension; import org.sonar.telemetry.core.Granularity; import org.sonar.telemetry.core.TelemetryDataProvider; @@ -60,8 +61,8 @@ public class TestTelemetryBean implements TelemetryDataProvider { } @Override - public String getValue() { - return METRIC_VALUE; + public Optional getValue() { + return Optional.of(METRIC_VALUE); } @Override diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/schema/InstallationMetricTest.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/schema/InstallationMetricTest.java index 43408b7a2ba..b3ecd01b7a0 100644 --- a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/schema/InstallationMetricTest.java +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/schema/InstallationMetricTest.java @@ -28,7 +28,7 @@ import static org.assertj.core.api.Assertions.assertThat; class InstallationMetricTest { @Test - void gettersAndSetters() { + void constructor() { InstallationMetric metric = new InstallationMetric( "installation-key-1", "value", @@ -42,4 +42,19 @@ class InstallationMetricTest { assertThat(metric.getType()).isEqualTo(TelemetryDataType.STRING); } + @Test + void constructor_shouldAcceptNullValue() { + InstallationMetric metric = new InstallationMetric( + "installation-key-1", + null, + TelemetryDataType.STRING, + Granularity.WEEKLY + ); + + assertThat(metric.getValue()).isNull(); + assertThat(metric.getKey()).isEqualTo("installation-key-1"); + assertThat(metric.getGranularity()).isEqualTo(Granularity.WEEKLY); + assertThat(metric.getType()).isEqualTo(TelemetryDataType.STRING); + } + } diff --git a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/util/SentMetricsStorageTest.java b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/util/SentMetricsStorageTest.java index ad2cf915d8b..804bda58bae 100644 --- a/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/util/SentMetricsStorageTest.java +++ b/server/sonar-telemetry/src/test/java/org/sonar/telemetry/metrics/util/SentMetricsStorageTest.java @@ -20,13 +20,13 @@ package org.sonar.telemetry.metrics.util; import com.tngtech.java.junit.dataprovider.DataProvider; -import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import org.sonar.api.impl.utils.TestSystem2; import org.sonar.db.telemetry.TelemetryMetricsSentDto; import org.sonar.telemetry.core.Dimension; import org.sonar.telemetry.core.Granularity; @@ -38,6 +38,8 @@ public class SentMetricsStorageTest { public static final String METRIC_3 = "metric-3"; public static final String METRIC_4 = "metric-4"; + private final TestSystem2 system2 = new TestSystem2().setNow(10_000_000_000L); + @DataProvider public static Object[][] data() { return new Object[][]{ @@ -64,7 +66,13 @@ public class SentMetricsStorageTest { // Non-existing metrics that should be sent, as they are sent for the first time {Dimension.INSTALLATION, "metric-5", Granularity.DAILY, true}, {Dimension.USER, "metric-6", Granularity.WEEKLY, true}, - {Dimension.PROJECT, "metric-7", Granularity.MONTHLY, true} + {Dimension.PROJECT, "metric-7", Granularity.MONTHLY, true}, + + // Adhoc granularity means the metric should ALWAYS be sent + {Dimension.INSTALLATION, "metric-8", Granularity.ADHOC, true}, + {Dimension.USER, "metric-9", Granularity.ADHOC, true}, + {Dimension.PROJECT, "metric-10", Granularity.ADHOC, true}, + {Dimension.LANGUAGE, "metric-11", Granularity.ADHOC, true} }; } @@ -72,22 +80,22 @@ public class SentMetricsStorageTest { @MethodSource("data") void shouldSendMetric(Dimension dimension, String metricKey, Granularity granularity, boolean expectedResult) { SentMetricsStorage storage = new SentMetricsStorage(getDtos()); - boolean actualResult = storage.shouldSendMetric(dimension, metricKey, granularity); + boolean actualResult = storage.shouldSendMetric(dimension, metricKey, granularity, system2.now()); Assertions.assertEquals(expectedResult, actualResult); } private List getDtos() { TelemetryMetricsSentDto dto1 = new TelemetryMetricsSentDto(METRIC_1, Dimension.INSTALLATION.getValue()); - dto1.setLastSent(Instant.now().minus(1, ChronoUnit.MINUTES).toEpochMilli()); // 1 minute ago + dto1.setLastSent(system2.now() - TimeUnit.MINUTES.toMillis(1)); // 1 minute ago TelemetryMetricsSentDto dto2 = new TelemetryMetricsSentDto(METRIC_2, Dimension.USER.getValue()); - dto2.setLastSent(Instant.now().minus(2, ChronoUnit.DAYS).toEpochMilli()); // 2 days ago + dto2.setLastSent(system2.now() - TimeUnit.DAYS.toMillis(2)); // 2 days ago TelemetryMetricsSentDto dto3 = new TelemetryMetricsSentDto(METRIC_3, Dimension.PROJECT.getValue()); - dto3.setLastSent(Instant.now().minus(10, ChronoUnit.DAYS).toEpochMilli()); // 10 days ago + dto3.setLastSent(system2.now() - TimeUnit.DAYS.toMillis(10)); // 10 days ago TelemetryMetricsSentDto dto4 = new TelemetryMetricsSentDto(METRIC_4, Dimension.LANGUAGE.getValue()); - dto4.setLastSent(Instant.now().minus(40, ChronoUnit.DAYS).toEpochMilli()); // 40 days ago + dto4.setLastSent(system2.now() - TimeUnit.DAYS.toMillis(40)); // 40 days ago return Arrays.asList(dto1, dto2, dto3, dto4); } diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/telemetry/TelemetryVersionProvider.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/telemetry/TelemetryVersionProvider.java index 850190948ba..101f694f99e 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/telemetry/TelemetryVersionProvider.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/telemetry/TelemetryVersionProvider.java @@ -19,6 +19,7 @@ */ package org.sonar.server.platform.telemetry; +import java.util.Optional; import org.sonar.api.platform.Server; import org.sonar.telemetry.core.Dimension; import org.sonar.telemetry.core.Granularity; @@ -54,7 +55,7 @@ public class TelemetryVersionProvider implements TelemetryDataProvider { } @Override - public String getValue() { - return server.getVersion(); + public Optional getValue() { + return Optional.ofNullable(server.getVersion()); } } diff --git a/server/sonar-webserver/src/test/java/org/sonar/server/platform/telemetry/TelemetryVersionProviderTest.java b/server/sonar-webserver/src/test/java/org/sonar/server/platform/telemetry/TelemetryVersionProviderTest.java index 1a14d52bbf3..d13295312b8 100644 --- a/server/sonar-webserver/src/test/java/org/sonar/server/platform/telemetry/TelemetryVersionProviderTest.java +++ b/server/sonar-webserver/src/test/java/org/sonar/server/platform/telemetry/TelemetryVersionProviderTest.java @@ -19,6 +19,7 @@ */ package org.sonar.server.platform.telemetry; +import java.util.Optional; import org.junit.jupiter.api.Test; import org.sonar.api.platform.Server; import org.sonar.telemetry.core.Dimension; @@ -46,7 +47,7 @@ class TelemetryVersionProviderTest { assertEquals(Dimension.INSTALLATION, telemetryVersionProvider.getDimension()); assertEquals(Granularity.DAILY, telemetryVersionProvider.getGranularity()); assertEquals(TelemetryDataType.STRING, telemetryVersionProvider.getType()); - assertEquals("10.6", telemetryVersionProvider.getValue()); + assertEquals(Optional.of("10.6"), telemetryVersionProvider.getValue()); assertThrows(IllegalStateException.class, telemetryVersionProvider::getUuidValues); } } -- 2.39.5