From 430fdc1da20095879d186a27aa6b3d20ef4dff04 Mon Sep 17 00:00:00 2001 From: Pierre Guillot <50145663+pierre-guillot-sonarsource@users.noreply.github.com> Date: Mon, 24 Jun 2019 10:51:58 +0200 Subject: [PATCH] SONAR-11304 add date and version of initial installation to telemetry (#1751) SONAR-11304 add date and version of initial installation to telemetry --- .../db/migration/version/v79/DbVersion79.java | 3 +- .../v79/PopulateInstallDateAndVersion.java | 88 ++++++++++++ .../migration/version/v79/package-info.java | 24 ++++ .../version/v79/DbVersion79Test.java | 2 +- .../PopulateInstallDateAndVersionTest.java | 126 ++++++++++++++++++ .../schema.sql | 39 ++++++ .../server/property/InternalProperties.java | 11 ++ .../db/migration/AutoDbMigration.java | 106 +++++++++------ .../sonar/server/telemetry/TelemetryData.java | 24 ++++ .../telemetry/TelemetryDataJsonWriter.java | 6 + .../server/telemetry/TelemetryDataLoader.java | 36 +++-- .../db/migration/AutoDbMigrationH2Test.java | 84 ++++++++++++ .../db/migration/AutoDbMigrationTest.java | 8 +- .../ws/ClusterSystemInfoWriterTest.java | 3 +- .../ws/StandaloneSystemInfoWriterTest.java | 2 +- .../server/telemetry/TelemetryDaemonTest.java | 30 ++++- 16 files changed, 534 insertions(+), 58 deletions(-) create mode 100644 server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersion.java create mode 100644 server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/package-info.java create mode 100644 server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersionTest.java create mode 100644 server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersionTest/schema.sql create mode 100644 server/sonar-server/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationH2Test.java diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79.java index 37242753450..ba6201566de 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79.java @@ -27,6 +27,7 @@ public class DbVersion79 implements DbVersion { public void addSteps(MigrationStepRegistry registry) { registry .add(2800, "Truncate environment variables and system properties from existing scanner reports", - TruncateEnvAndSystemVarsFromScannerContext.class); + TruncateEnvAndSystemVarsFromScannerContext.class) + .add(2801, "populate install version and install date internal properties", PopulateInstallDateAndVersion.class); } } diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersion.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersion.java new file mode 100644 index 00000000000..c967150ec49 --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersion.java @@ -0,0 +1,88 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v79; + +import java.sql.SQLException; +import org.sonar.api.SonarRuntime; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.db.Database; +import org.sonar.server.platform.db.migration.step.DataChange; + +public class PopulateInstallDateAndVersion extends DataChange { + + private static final Logger LOG = Loggers.get(PopulateInstallDateAndVersion.class); + private static final String INSTALLATION_DATE = "installation.date"; + private static final String INSTALLATION_VERSION = "installation.version"; + private final SonarRuntime sonarRuntime; + private final System2 system2; + + public PopulateInstallDateAndVersion(Database db, SonarRuntime sonarRuntime, System2 system2) { + super(db); + this.sonarRuntime = sonarRuntime; + this.system2 = system2; + } + + @Override + protected void execute(Context context) throws SQLException { + removeProperties(context); + Long createdAt = context.prepareSelect("select min(created_at) from users where created_at is not null") + .get(row -> row.getLong(1)); + if (createdAt != null && createdAt != 0) { + populateInstallationDate(context, createdAt); + populateInstallationVersion(context, createdAt); + } + } + + private void populateInstallationDate(Context context, Long createdAt) throws SQLException { + insertInternalProperty(context, INSTALLATION_DATE, String.valueOf(createdAt)); + } + + private void populateInstallationVersion(Context context, Long createdAt) throws SQLException { + if (Math.abs(system2.now() - createdAt) / 60 / 60 / 1000 <= 24) { + String apiVersion = sonarRuntime.getApiVersion().toString(); + insertInternalProperty(context, INSTALLATION_VERSION, apiVersion); + } else { + // if the difference between now and smallest account creation date is more than a day, we consider that this is a + // start with an existing SQ, and not a fresh start. in this case, we do not populate the internalProperty, + // as there is no way to know the original SQ installation version. + LOG.warn("skipping " + INSTALLATION_VERSION + " because we cannot determine what is the installation version."); + } + } + + private void insertInternalProperty(Context context, String key, String value) throws SQLException { + context.prepareUpsert("insert into internal_properties (kee, is_empty, text_value, clob_value, created_at) VALUES (?, ?, ?, ?, ?)") + .setString(1, key) + .setBoolean(2, false) + .setString(3, value) + .setString(4, null) + .setLong(5, system2.now()) + .execute().commit().close(); + } + + private static void removeProperties(Context context) throws SQLException { + context.prepareUpsert("delete from internal_properties where kee = ? or kee = ?") + .setString(1, INSTALLATION_DATE) + .setString(2, INSTALLATION_VERSION) + .execute().commit().close(); + + } +} diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/package-info.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/package-info.java new file mode 100644 index 00000000000..1bcbde154dd --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/package-info.java @@ -0,0 +1,24 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.server.platform.db.migration.version.v79; + +import javax.annotation.ParametersAreNonnullByDefault; + diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79Test.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79Test.java index dc06cf84f8a..ad2680616cc 100644 --- a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79Test.java +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79Test.java @@ -35,7 +35,7 @@ public class DbVersion79Test { @Test public void verify_migration_count() { - verifyMigrationCount(underTest, 1); + verifyMigrationCount(underTest, 2); } } diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersionTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersionTest.java new file mode 100644 index 00000000000..5944bb29f01 --- /dev/null +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersionTest.java @@ -0,0 +1,126 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v79; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import org.apache.commons.lang.math.RandomUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.SonarRuntime; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.Version; +import org.sonar.db.CoreDbTester; +import org.sonar.server.platform.db.migration.step.DataChange; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.groups.Tuple.tuple; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PopulateInstallDateAndVersionTest { + + @Rule + public CoreDbTester db = CoreDbTester.createForSchema(PopulateInstallDateAndVersionTest.class, "schema.sql"); + + private System2 system2 = mock(System2.class); + private SonarRuntime sonarRuntime = mock(SonarRuntime.class); + + private DataChange underTest = new PopulateInstallDateAndVersion(db.database(), sonarRuntime, system2); + + private Function, Object> field(String name) { + return m -> m.get(name); + } + + @Before + public void before() { + Version version = Version.create(7, 9, 0); + when(sonarRuntime.getApiVersion()).thenReturn(version); + when(system2.now()).thenReturn(RandomUtils.nextLong()); + truncateUsers(); + truncateInternalProperties(); + } + + @Test + public void migrateFreshInstall() throws SQLException { + Long createdAt = system2.now() - (23 * 60 * 60 * 1000); + insertAdminUser(createdAt); + + underTest.execute(); + + assertThat(db.select("select * from internal_properties")).extracting( + field("CREATED_AT"), field("CLOB_VALUE"), + field("KEE"), field("TEXT_VALUE"), field("IS_EMPTY")) + .containsExactlyInAnyOrder( + tuple(system2.now(), null, "installation.date", String.valueOf(createdAt), false), + tuple(system2.now(), null, "installation.version", "7.9", false)); + } + + @Test + public void migrateOldInstance() throws SQLException { + Long createdAt = system2.now() - (25 * 60 * 60 * 1000); + insertAdminUser(createdAt); + + underTest.execute(); + + assertThat(db.select("select * from internal_properties")).extracting( + field("CREATED_AT"), field("CLOB_VALUE"), + field("KEE"), field("TEXT_VALUE"), field("IS_EMPTY")) + .containsExactlyInAnyOrder( + tuple(system2.now(), null, "installation.date", String.valueOf(createdAt), false)); + } + + @Test + public void migrateNoUsers() throws SQLException { + underTest.execute(); + + assertThat(db.select("select * from internal_properties").stream().count()).isEqualTo(0); + } + + private void insertAdminUser(long createdAt) { + Map values = new HashMap<>(); + values.put("UUID", "UUID"); + values.put("login", "admin"); + values.put("name", "Administrator"); + values.put("email", null); + values.put("EXTERNAL_ID", "admin"); + values.put("EXTERNAL_LOGIN", "admin"); + values.put("external_identity_provider", "sonarqube"); + values.put("user_local", true); + values.put("crypted_password", "a373a0e667abb2604c1fd571eb4ad47fe8cc0878"); + values.put("salt", "48bc4b0d93179b5103fd3885ea9119498e9d161b"); + values.put("created_at", createdAt); + values.put("updated_at", createdAt); + values.put("IS_ROOT", true); + values.put("ONBOARDED", false); + db.executeInsert("users", values); + } + + private void truncateUsers() { + db.executeUpdateSql("truncate table users"); + } + + private void truncateInternalProperties() { + db.executeUpdateSql("truncate table internal_properties"); + } +} diff --git a/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersionTest/schema.sql b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersionTest/schema.sql new file mode 100644 index 00000000000..67390214cd8 --- /dev/null +++ b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v79/PopulateInstallDateAndVersionTest/schema.sql @@ -0,0 +1,39 @@ +CREATE TABLE "INTERNAL_PROPERTIES" ( + "KEE" VARCHAR(20) NOT NULL, + "IS_EMPTY" BOOLEAN NOT NULL, + "TEXT_VALUE" VARCHAR(4000), + "CLOB_VALUE" CLOB, + "CREATED_AT" BIGINT, + + CONSTRAINT "PK_INTERNAL_PROPERTIES" PRIMARY KEY ("KEE") +); + +CREATE TABLE "USERS" ( + "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "UUID" VARCHAR(255) NOT NULL, + "LOGIN" VARCHAR(255) NOT NULL, + "NAME" VARCHAR(200), + "EMAIL" VARCHAR(100), + "CRYPTED_PASSWORD" VARCHAR(100), + "SALT" VARCHAR(40), + "HASH_METHOD" VARCHAR(10), + "ACTIVE" BOOLEAN DEFAULT TRUE, + "SCM_ACCOUNTS" VARCHAR(4000), + "EXTERNAL_ID" VARCHAR(255) NOT NULL, + "EXTERNAL_LOGIN" VARCHAR(255) NOT NULL, + "EXTERNAL_IDENTITY_PROVIDER" VARCHAR(100) NOT NULL, + "IS_ROOT" BOOLEAN NOT NULL, + "USER_LOCAL" BOOLEAN, + "ONBOARDED" BOOLEAN NOT NULL, + "HOMEPAGE_TYPE" VARCHAR(40), + "HOMEPAGE_PARAMETER" VARCHAR(40), + "ORGANIZATION_UUID" VARCHAR(40), + "LAST_CONNECTION_DATE" BIGINT, + "CREATED_AT" BIGINT, + "UPDATED_AT" BIGINT +); +CREATE UNIQUE INDEX "USERS_UUID" ON "USERS" ("UUID"); +CREATE UNIQUE INDEX "USERS_LOGIN" ON "USERS" ("LOGIN"); +CREATE UNIQUE INDEX "UNIQ_EXTERNAL_ID" ON "USERS" ("EXTERNAL_IDENTITY_PROVIDER", "EXTERNAL_ID"); +CREATE UNIQUE INDEX "UNIQ_EXTERNAL_LOGIN" ON "USERS" ("EXTERNAL_IDENTITY_PROVIDER", "EXTERNAL_LOGIN"); +CREATE INDEX "USERS_UPDATED_AT" ON "USERS" ("UPDATED_AT"); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/property/InternalProperties.java b/server/sonar-server-common/src/main/java/org/sonar/server/property/InternalProperties.java index d4bb37963a3..1eade4a51a7 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/property/InternalProperties.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/property/InternalProperties.java @@ -42,6 +42,17 @@ public interface InternalProperties { String COMPUTE_ENGINE_PAUSE = "ce.pause"; String BITBUCKETCLOUD_APP_SHAREDSECRET = "bbc.app.sharedSecret"; + + /** + * First installation date + */ + String INSTALLATION_DATE = "installation.date"; + + /** + * first installation SQ version + */ + String INSTALLATION_VERSION = "installation.version"; + /** * Read the value of the specified property. * diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/db/migration/AutoDbMigration.java b/server/sonar-server/src/main/java/org/sonar/server/platform/db/migration/AutoDbMigration.java index 8a74f9dabe4..bac37c6cefe 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/db/migration/AutoDbMigration.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/db/migration/AutoDbMigration.java @@ -25,6 +25,8 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import org.apache.commons.dbutils.DbUtils; import org.picocontainer.Startable; +import org.sonar.api.SonarRuntime; +import org.sonar.api.utils.System2; import org.sonar.api.utils.log.Loggers; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -35,6 +37,9 @@ import org.sonar.server.platform.DefaultServerUpgradeStatus; import org.sonar.server.platform.db.migration.engine.MigrationEngine; import org.sonar.server.platform.db.migration.step.MigrationSteps; +import static org.sonar.server.property.InternalProperties.INSTALLATION_DATE; +import static org.sonar.server.property.InternalProperties.INSTALLATION_VERSION; + /** * FIXME fix this class to remove use of DdlUtils.createSchema */ @@ -43,46 +48,17 @@ public class AutoDbMigration implements Startable { private final DbClient dbClient; private final MigrationEngine migrationEngine; private final MigrationSteps migrationSteps; + private final SonarRuntime sonarRuntime; + private final System2 system2; - public AutoDbMigration(DefaultServerUpgradeStatus serverUpgradeStatus, DbClient dbClient, MigrationEngine migrationEngine, MigrationSteps migrationSteps) { + public AutoDbMigration(DefaultServerUpgradeStatus serverUpgradeStatus, DbClient dbClient, MigrationEngine migrationEngine, MigrationSteps migrationSteps, + SonarRuntime sonarRuntime, System2 system2) { this.serverUpgradeStatus = serverUpgradeStatus; this.dbClient = dbClient; this.migrationEngine = migrationEngine; this.migrationSteps = migrationSteps; - } - - @Override - public void start() { - if (serverUpgradeStatus.isFreshInstall()) { - Loggers.get(getClass()).info("Automatically perform DB migration on fresh install"); - Dialect dialect = dbClient.getDatabase().getDialect(); - if (H2.ID.equals(dialect.getId())) { - installH2(); - } else { - migrationEngine.execute(); - } - } else if (serverUpgradeStatus.isUpgraded() && serverUpgradeStatus.isBlueGreen()) { - Loggers.get(getClass()).info("Automatically perform DB migration on blue/green deployment"); - migrationEngine.execute(); - } - } - - @VisibleForTesting - void installH2() { - Connection connection = null; - try (DbSession session = dbClient.openSession(false)) { - connection = session.getConnection(); - createH2Schema(connection, dbClient.getDatabase().getDialect().getId()); - } finally { - DbUtils.closeQuietly(connection); - } - } - - @VisibleForTesting - protected void createH2Schema(Connection connection, String dialectId) { - DdlUtils.createSchema(connection, dialectId, false); - populateSchemaMigration(connection, migrationSteps.getMaxMigrationNumber()); - hackFixForProjectMeasureTreeQueries(connection); + this.sonarRuntime = sonarRuntime; + this.system2 = system2; } private static void populateSchemaMigration(Connection connection, long maxMigrationNumber) { @@ -134,13 +110,67 @@ public class AutoDbMigration implements Startable { connection.commit(); } - @FunctionalInterface - private interface Preparer { - void prepare(PreparedStatement statement, long counter) throws SQLException; + @Override + public void start() { + if (serverUpgradeStatus.isFreshInstall()) { + Loggers.get(getClass()).info("Automatically perform DB migration on fresh install"); + Dialect dialect = dbClient.getDatabase().getDialect(); + if (H2.ID.equals(dialect.getId())) { + installH2(); + } else { + migrationEngine.execute(); + } + } else if (serverUpgradeStatus.isUpgraded() && serverUpgradeStatus.isBlueGreen()) { + Loggers.get(getClass()).info("Automatically perform DB migration on blue/green deployment"); + migrationEngine.execute(); + } + } + + @VisibleForTesting + void installH2() { + Connection connection = null; + try (DbSession session = dbClient.openSession(false)) { + connection = session.getConnection(); + createH2Schema(connection, dbClient.getDatabase().getDialect().getId()); + } finally { + DbUtils.closeQuietly(connection); + } + } + + @VisibleForTesting + protected void createH2Schema(Connection connection, String dialectId) { + DdlUtils.createSchema(connection, dialectId, false); + populateInstallDateAndVersion(connection); + populateSchemaMigration(connection, migrationSteps.getMaxMigrationNumber()); + hackFixForProjectMeasureTreeQueries(connection); + } + + private void populateInstallDateAndVersion(Connection connection) { + insertInternalProperty(connection, INSTALLATION_DATE, String.valueOf(system2.now())); + insertInternalProperty(connection, INSTALLATION_VERSION, sonarRuntime.getApiVersion().toString()); + } + + private void insertInternalProperty(Connection connection, String key, String value) { + try (PreparedStatement preparedStatementDate = connection + .prepareStatement("insert into internal_properties (kee, is_empty, text_value, clob_value, created_at) values (?, ?, ?, ?, ?)")) { + preparedStatementDate.setString(1, key); + preparedStatementDate.setBoolean(2, false); + preparedStatementDate.setString(3, value); + preparedStatementDate.setString(4, null); + preparedStatementDate.setLong(5, system2.now()); + preparedStatementDate.execute(); + } catch (SQLException e) { + throw new RuntimeException("Failed to insert internal properties " + key, e); + } } @Override public void stop() { // nothing to do } + + @FunctionalInterface + private interface Preparer { + void prepare(PreparedStatement statement, long counter) throws SQLException; + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryData.java b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryData.java index 5d3ebc80a15..89f3df4b38f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryData.java +++ b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryData.java @@ -40,6 +40,8 @@ public class TelemetryData { private final Map nclocByLanguage; private final Optional edition; private final String licenseType; + private final Long installationDate; + private final String installationVersion; private TelemetryData(Builder builder) { serverId = builder.serverId; @@ -54,6 +56,8 @@ public class TelemetryData { nclocByLanguage = builder.projectMeasuresStatistics.getNclocByLanguage(); edition = builder.edition; licenseType = builder.licenseType; + installationDate = builder.installationDate; + installationVersion = builder.installationVersion; } public String getServerId() { @@ -104,6 +108,14 @@ public class TelemetryData { return Optional.ofNullable(licenseType); } + public Long getInstallationDate(){ + return installationDate; + } + + public String getInstallationVersion(){ + return installationVersion; + } + static Builder builder() { return new Builder(); } @@ -119,6 +131,8 @@ public class TelemetryData { private Boolean usingBranches; private Optional edition; private String licenseType; + private Long installationDate; + private String installationVersion; private Builder() { // enforce static factory method @@ -174,6 +188,16 @@ public class TelemetryData { return this; } + public Builder setInstallationDate(@Nullable Long installationDate){ + this.installationDate = installationDate; + return this; + } + + public Builder setInstallationVersion(@Nullable String installationVersion){ + this.installationVersion = installationVersion; + return this; + } + TelemetryData build() { requireNonNull(serverId); requireNonNull(version); diff --git a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java index 0896e45ea19..8f4d17696d5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java +++ b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java @@ -71,6 +71,12 @@ public class TelemetryDataJsonWriter { json.endObject(); }); json.endArray(); + if (statistics.getInstallationDate() != null) { + json.prop("installationDate", statistics.getInstallationDate()); + } + if (statistics.getInstallationVersion() != null) { + json.prop("installationVersion", statistics.getInstallationVersion()); + } json.endObject(); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDataLoader.java b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDataLoader.java index 071d9e249ac..78adb7e6687 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDataLoader.java +++ b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryDataLoader.java @@ -22,6 +22,7 @@ package org.sonar.server.telemetry; import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.util.Map; +import java.util.Optional; import java.util.function.Function; import javax.annotation.CheckForNull; import javax.annotation.Nullable; @@ -38,6 +39,7 @@ import org.sonar.server.es.SearchOptions; import org.sonar.server.measure.index.ProjectMeasuresIndex; import org.sonar.server.measure.index.ProjectMeasuresStatistics; import org.sonar.server.organization.DefaultOrganizationProvider; +import org.sonar.server.property.InternalProperties; import org.sonar.server.telemetry.TelemetryData.Database; import org.sonar.server.user.index.UserIndex; import org.sonar.server.user.index.UserQuery; @@ -53,16 +55,18 @@ public class TelemetryDataLoader { private final ProjectMeasuresIndex projectMeasuresIndex; private final PlatformEditionProvider editionProvider; private final DefaultOrganizationProvider defaultOrganizationProvider; + private final InternalProperties internalProperties; @CheckForNull private final LicenseReader licenseReader; public TelemetryDataLoader(Server server, DbClient dbClient, PluginRepository pluginRepository, UserIndex userIndex, ProjectMeasuresIndex projectMeasuresIndex, - PlatformEditionProvider editionProvider, DefaultOrganizationProvider defaultOrganizationProvider) { - this(server, dbClient, pluginRepository, userIndex, projectMeasuresIndex, editionProvider, defaultOrganizationProvider, null); + PlatformEditionProvider editionProvider, DefaultOrganizationProvider defaultOrganizationProvider, InternalProperties internalProperties) { + this(server, dbClient, pluginRepository, userIndex, projectMeasuresIndex, editionProvider, defaultOrganizationProvider, internalProperties, null); } public TelemetryDataLoader(Server server, DbClient dbClient, PluginRepository pluginRepository, UserIndex userIndex, ProjectMeasuresIndex projectMeasuresIndex, - PlatformEditionProvider editionProvider, DefaultOrganizationProvider defaultOrganizationProvider, @Nullable LicenseReader licenseReader) { + PlatformEditionProvider editionProvider, DefaultOrganizationProvider defaultOrganizationProvider, InternalProperties internalProperties, + @Nullable LicenseReader licenseReader) { this.server = server; this.dbClient = dbClient; this.pluginRepository = pluginRepository; @@ -71,6 +75,16 @@ public class TelemetryDataLoader { this.editionProvider = editionProvider; this.defaultOrganizationProvider = defaultOrganizationProvider; this.licenseReader = licenseReader; + this.internalProperties = internalProperties; + } + + private static Database loadDatabaseMetadata(DbSession dbSession) { + try { + DatabaseMetaData metadata = dbSession.getConnection().getMetaData(); + return new Database(metadata.getDatabaseProductName(), metadata.getDatabaseProductVersion()); + } catch (SQLException e) { + throw new IllegalStateException("Fail to get DB metadata", e); + } } public TelemetryData load() { @@ -99,19 +113,17 @@ public class TelemetryDataLoader { data.setNcloc(dbClient.liveMeasureDao().sumNclocOfBiggestLongLivingBranch(dbSession, query)); } + Optional installationDateProperty = internalProperties.read(InternalProperties.INSTALLATION_DATE); + if (installationDateProperty.isPresent()) { + data.setInstallationDate(Long.valueOf(installationDateProperty.get())); + } + Optional installationVersionProperty = internalProperties.read(InternalProperties.INSTALLATION_VERSION); + data.setInstallationVersion(installationVersionProperty.orElse(null)); + return data.build(); } String loadServerId() { return server.getId(); } - - private static Database loadDatabaseMetadata(DbSession dbSession) { - try { - DatabaseMetaData metadata = dbSession.getConnection().getMetaData(); - return new Database(metadata.getDatabaseProductName(), metadata.getDatabaseProductVersion()); - } catch (SQLException e) { - throw new IllegalStateException("Fail to get DB metadata", e); - } - } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationH2Test.java b/server/sonar-server/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationH2Test.java new file mode 100644 index 00000000000..991eaf56f0b --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationH2Test.java @@ -0,0 +1,84 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.mockito.Mockito; +import org.sonar.api.SonarRuntime; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.Version; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.H2Database; +import org.sonar.db.dialect.H2; +import org.sonar.server.platform.DefaultServerUpgradeStatus; +import org.sonar.server.platform.db.migration.engine.MigrationEngine; +import org.sonar.server.platform.db.migration.history.MigrationHistoryTableImpl; +import org.sonar.server.platform.db.migration.step.MigrationSteps; +import org.sonar.server.property.InternalProperties; + +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AutoDbMigrationH2Test { + + private DbClient dbClient = mock(DbClient.class, Mockito.RETURNS_DEEP_STUBS); + private DefaultServerUpgradeStatus serverUpgradeStatus = mock(DefaultServerUpgradeStatus.class); + private MigrationEngine migrationEngine = mock(MigrationEngine.class); + private MigrationSteps migrationSteps = mock(MigrationSteps.class); + private SonarRuntime sonarRuntime = mock(SonarRuntime.class); + private System2 system2 = mock(System2.class); + + private AutoDbMigration underTest = new AutoDbMigration(serverUpgradeStatus, dbClient, migrationEngine, migrationSteps, sonarRuntime, system2); + + + @Test + public void testInstallH2() throws SQLException { + DbSession dbSession = mock(DbSession.class); + when(dbClient.getDatabase().getDialect()).thenReturn(new H2()); + when(dbClient.openSession(anyBoolean())).thenReturn(dbSession); + when(system2.now()).thenReturn(123456789L); + H2Database db = new H2Database("sonar", false); + db.start(); + Connection connection = db.getDataSource().getConnection(); + when(dbSession.getConnection()).thenReturn(connection); + Version version = Version.create(7, 9, 0); + when(sonarRuntime.getApiVersion()).thenReturn(version); + new MigrationHistoryTableImpl(db).start(); + + underTest.installH2(); + + String selectInstallVersion = "select text_value from internal_properties where kee = '" + InternalProperties.INSTALLATION_VERSION + "'"; + ResultSet resultSetVersion = db.getDataSource().getConnection().prepareStatement(selectInstallVersion).executeQuery(); + resultSetVersion.next(); + Assertions.assertThat(resultSetVersion.getString(1)).isEqualTo("7.9"); + + String selectInstallDate = "select text_value from internal_properties where kee = '" + InternalProperties.INSTALLATION_DATE + "'"; + ResultSet resultSetDate = db.getDataSource().getConnection().prepareStatement(selectInstallDate).executeQuery(); + resultSetDate.next(); + Assertions.assertThat(resultSetDate.getString(1)).isEqualTo("123456789"); + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationTest.java index 785a684acff..460b6576898 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationTest.java @@ -24,6 +24,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mockito; +import org.sonar.api.SonarRuntime; +import org.sonar.api.utils.System2; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.db.DbClient; @@ -56,9 +58,11 @@ public class AutoDbMigrationTest { private DefaultServerUpgradeStatus serverUpgradeStatus = mock(DefaultServerUpgradeStatus.class); private MigrationEngine migrationEngine = mock(MigrationEngine.class); private MigrationSteps migrationSteps = mock(MigrationSteps.class); - private AutoDbMigration underTest = new AutoDbMigration(serverUpgradeStatus, dbClient, migrationEngine, migrationSteps); + private SonarRuntime sonarRuntime = mock(SonarRuntime.class); + private System2 system2 = mock(System2.class); + private AutoDbMigration underTest = new AutoDbMigration(serverUpgradeStatus, dbClient, migrationEngine, migrationSteps, sonarRuntime, system2); - private AutoDbMigration noRealH2Creation = spy(new AutoDbMigration(serverUpgradeStatus, dbClient, migrationEngine, migrationSteps) { + private AutoDbMigration noRealH2Creation = spy(new AutoDbMigration(serverUpgradeStatus, dbClient, migrationEngine, migrationSteps, sonarRuntime, system2) { @Override protected void createH2Schema(Connection connection, String dialectId) { // do nothing diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/ws/ClusterSystemInfoWriterTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/ws/ClusterSystemInfoWriterTest.java index b6d0d983221..777365d2b62 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/ws/ClusterSystemInfoWriterTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/ws/ClusterSystemInfoWriterTest.java @@ -70,7 +70,8 @@ public class ClusterSystemInfoWriterTest { + "\"Application Nodes\":[{\"Name\":\"appNodes\",\"\":{\"name\":\"appNodes\"}}]," + "\"Search Nodes\":[{\"Name\":\"searchNodes\",\"\":{\"name\":\"searchNodes\"}}]," + "\"Statistics\":{\"id\":\"\",\"version\":\"\",\"database\":{\"name\":\"\",\"version\":\"\"},\"plugins\":[]," - + "\"userCount\":0,\"projectCount\":0,\"usingBranches\":false,\"ncloc\":0,\"projectCountByLanguage\":[],\"nclocByLanguage\":[]}}"); + + "\"userCount\":0,\"projectCount\":0,\"usingBranches\":false,\"ncloc\":0,\"projectCountByLanguage\":[]," + + "\"nclocByLanguage\":[],\"installationDate\":0,\"installationVersion\":\"\"}}"); } private static NodeInfo createNodeInfo(String name) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/ws/StandaloneSystemInfoWriterTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/ws/StandaloneSystemInfoWriterTest.java index 5a17b5f42cd..ad7e1edd45c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/ws/StandaloneSystemInfoWriterTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/ws/StandaloneSystemInfoWriterTest.java @@ -79,7 +79,7 @@ public class StandaloneSystemInfoWriterTest { // response does not contain empty "Section Three" assertThat(writer.toString()).isEqualTo("{\"Health\":\"GREEN\",\"Health Causes\":[],\"Section One\":{\"foo\":\"bar\"},\"Section Two\":{\"one\":1,\"two\":2}," + "\"Statistics\":{\"id\":\"\",\"version\":\"\",\"database\":{\"name\":\"\",\"version\":\"\"},\"plugins\":[],\"userCount\":0,\"projectCount\":0,\"usingBranches\":false," + - "\"ncloc\":0,\"projectCountByLanguage\":[],\"nclocByLanguage\":[]}}"); + "\"ncloc\":0,\"projectCountByLanguage\":[],\"nclocByLanguage\":[],\"installationDate\":0,\"installationVersion\":\"\"}}"); } private void logInAsSystemAdministrator() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java b/server/sonar-server/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java index 01a78fb718b..82c1cc48384 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java @@ -102,12 +102,12 @@ public class TelemetryDaemonTest { private PlatformEditionProvider editionProvider = mock(PlatformEditionProvider.class); private final TelemetryDataLoader communityDataLoader = new TelemetryDataLoader(server, db.getDbClient(), pluginRepository, new UserIndex(es.client(), system2), - new ProjectMeasuresIndex(es.client(), null, system2), editionProvider, new DefaultOrganizationProviderImpl(db.getDbClient()), null); + new ProjectMeasuresIndex(es.client(), null, system2), editionProvider, new DefaultOrganizationProviderImpl(db.getDbClient()), internalProperties, null); private TelemetryDaemon communityUnderTest = new TelemetryDaemon(communityDataLoader, client, settings.asConfig(), internalProperties, system2); private final LicenseReader licenseReader = mock(LicenseReader.class); private final TelemetryDataLoader commercialDataLoader = new TelemetryDataLoader(server, db.getDbClient(), pluginRepository, new UserIndex(es.client(), system2), - new ProjectMeasuresIndex(es.client(), null, system2), editionProvider, new DefaultOrganizationProviderImpl(db.getDbClient()), licenseReader); + new ProjectMeasuresIndex(es.client(), null, system2), editionProvider, new DefaultOrganizationProviderImpl(db.getDbClient()), internalProperties, licenseReader); private TelemetryDaemon commercialUnderTest = new TelemetryDaemon(commercialDataLoader, client, settings.asConfig(), internalProperties, system2); @After @@ -272,6 +272,32 @@ public class TelemetryDaemonTest { assertThat(json.getValue()).contains(id, version); } + @Test + public void send_server_installation_date_and_installation_version() throws IOException { + initTelemetrySettingsToDefaultValues(); + settings.setProperty("sonar.telemetry.frequencyInSeconds", "1"); + String installationVersion = "7.9.BEST.LTS.EVER"; + Long installationDate = 1546300800000L; // 2019/01/01 + internalProperties.write(InternalProperties.INSTALLATION_DATE, String.valueOf(installationDate)); + internalProperties.write(InternalProperties.INSTALLATION_VERSION, installationVersion); + + communityUnderTest.start(); + + ArgumentCaptor json = captureJson(); + assertThat(json.getValue()).contains(installationVersion, installationDate.toString()); + } + + @Test + public void do_not_send_server_installation_details_if_missing_property() throws IOException { + initTelemetrySettingsToDefaultValues(); + settings.setProperty("sonar.telemetry.frequencyInSeconds", "1"); + + communityUnderTest.start(); + + ArgumentCaptor json = captureJson(); + assertThat(json.getValue()).doesNotContain("installationVersion", "installationDate"); + } + @Test public void do_not_send_data_if_last_ping_earlier_than_one_week_ago() throws IOException { initTelemetrySettingsToDefaultValues(); -- 2.39.5