From dc12e96975f798ba02f90f345fc81b8e7aeffa3c Mon Sep 17 00:00:00 2001 From: Michal Duda Date: Tue, 27 Nov 2018 14:24:13 +0100 Subject: [PATCH] SONAR-11523 migrate module lvl properties Move module level properties to a single project level property --- .../db/migration/version/v75/DbVersion75.java | 2 +- .../version/v75/MigrateModuleProperties.java | 119 ++++++++++++++ .../version/v75/DbVersion75Test.java | 2 +- .../v75/MigrateModulePropertiesTest.java | 153 ++++++++++++++++++ .../MigrateModulePropertiesTest/schema.sql | 35 ++++ 5 files changed, 309 insertions(+), 2 deletions(-) create mode 100644 server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v75/MigrateModuleProperties.java create mode 100644 server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v75/MigrateModulePropertiesTest.java create mode 100644 server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v75/MigrateModulePropertiesTest/schema.sql diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v75/DbVersion75.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v75/DbVersion75.java index 971b2cf740b..fd5f6716470 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v75/DbVersion75.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v75/DbVersion75.java @@ -32,6 +32,6 @@ public class DbVersion75 implements DbVersion { .add(2402, "Add column USER_EXTERNAL_ID in ALM_APP_INSTALLS", AddUserExternalIdColumnInAlmAppInstall.class) .add(2403, "Set IS_OWNER_USER not nullable in ALM_APP_INSTALLS", SetIsOwnerUserNotNullableInAlmAppInstalls.class) .add(2404, "Add table EVENT_COMPONENT_CHANGES", AddEventComponentChanges.class) - ; + .add(2405, "Move module and directory properties to a project level property", MigrateModuleProperties.class); } } diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v75/MigrateModuleProperties.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v75/MigrateModuleProperties.java new file mode 100644 index 00000000000..5a7d66c3877 --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v75/MigrateModuleProperties.java @@ -0,0 +1,119 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v75; + +import java.sql.SQLException; +import java.util.concurrent.atomic.AtomicReference; +import org.sonar.api.utils.System2; +import org.sonar.db.Database; +import org.sonar.server.platform.db.migration.SupportsBlueGreen; +import org.sonar.server.platform.db.migration.step.DataChange; +import org.sonar.server.platform.db.migration.step.MassUpdate; +import org.sonar.server.platform.db.migration.step.Upsert; + +@SupportsBlueGreen +public class MigrateModuleProperties extends DataChange { + + private static final String NEW_PROPERTY_NAME = "sonar.subprojects.settings.removed"; + + private final System2 system2; + + public MigrateModuleProperties(Database db, System2 system2) { + super(db); + this.system2 = system2; + } + + @Override + protected void execute(Context context) throws SQLException { + long now = system2.now(); + moveModulePropertiesToProjectLevel(context, now); + removeModuleProperties(context); + } + + private static void moveModulePropertiesToProjectLevel(Context context, long time) throws SQLException { + StringBuilder builder = new StringBuilder(); + AtomicReference currentProjectId = new AtomicReference<>(); + AtomicReference currentModuleUuid = new AtomicReference<>(); + + context.prepareSelect("select prop.prop_key, prop.text_value, prop.clob_value, mod.name, mod.uuid, root.id as project_id, root.name as project_name " + + "from properties prop " + + "left join projects mod on mod.id = prop.resource_id " + + "left join projects root on root.uuid = mod.project_uuid " + + "where mod.qualifier = 'BRC'" + + "order by root.uuid, mod.uuid, prop.prop_key") + .scroll(row -> { + String propertyKey = row.getString(1); + String propertyTextValue = row.getString(2); + String propertyClobValue = row.getString(3); + String moduleName = row.getString(4); + String moduleUuid = row.getString(5); + Integer projectId = row.getInt(6); + String projectName = row.getString(7); + + if (!projectId.equals(currentProjectId.get())) { + if (currentProjectId.get() != null) { + insertProjectProperties(context, currentProjectId.get(), builder.toString(), time); + } + + builder.setLength(0); + currentProjectId.set(projectId); + } + + if (!moduleUuid.equals(currentModuleUuid.get())) { + if (currentModuleUuid.get() != null && builder.length() != 0) { + builder.append("\n"); + } + builder.append("# previous settings for sub-project ").append(projectName).append("::").append(moduleName).append("\n"); + currentModuleUuid.set(moduleUuid); + } + + String propertyValue = propertyTextValue == null ? propertyClobValue : propertyTextValue; + builder.append(propertyKey).append("=").append(propertyValue).append("\n"); + }); + + if (builder.length() > 0) { + insertProjectProperties(context, currentProjectId.get(), builder.toString(), time); + } + } + + private static void insertProjectProperties(Context context, int projectId, String content, long time) throws SQLException { + Upsert upsert = context.prepareUpsert("insert into properties (prop_key, resource_id, is_empty, clob_value, created_at) values (?, ?, ?, ?, ?)"); + upsert.setString(1, NEW_PROPERTY_NAME) + .setInt(2, projectId) + .setBoolean(3, false) + .setString(4, content) + .setLong(5, time) + .execute() + .commit(); + } + + private static void removeModuleProperties(Context context) throws SQLException { + MassUpdate massUpdate = context.prepareMassUpdate().rowPluralName("remove module properties"); + massUpdate.select("select prop.id as property_id " + + "from properties prop " + + "left join projects mod on mod.id = prop.resource_id " + + "where mod.qualifier = 'BRC'"); + massUpdate.update("delete from properties where id=?"); + massUpdate.execute((row, update) -> { + update.setInt(1, row.getInt(1)); + return true; + }); + } +} diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v75/DbVersion75Test.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v75/DbVersion75Test.java index 6a0eee289e2..1d8a5afbba6 100644 --- a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v75/DbVersion75Test.java +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v75/DbVersion75Test.java @@ -35,6 +35,6 @@ public class DbVersion75Test { @Test public void verify_migration_count() { - verifyMigrationCount(underTest, 5); + verifyMigrationCount(underTest, 6); } } diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v75/MigrateModulePropertiesTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v75/MigrateModulePropertiesTest.java new file mode 100644 index 00000000000..f13c80dcf8e --- /dev/null +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v75/MigrateModulePropertiesTest.java @@ -0,0 +1,153 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v75; + +import com.google.common.collect.ImmutableSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Scopes; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.internal.TestSystem2; +import org.sonar.core.util.SequenceUuidFactory; +import org.sonar.core.util.UuidFactory; +import org.sonar.db.CoreDbTester; + +import static java.lang.String.valueOf; +import static org.assertj.core.api.Assertions.assertThat; + +public class MigrateModulePropertiesTest { + private final static long NOW = 50_000_000_000L; + + @Rule + public CoreDbTester db = CoreDbTester.createForSchema(MigrateModulePropertiesTest.class, "schema.sql"); + private System2 system2 = new TestSystem2().setNow(NOW); + private UuidFactory uuidFactory = new SequenceUuidFactory(); + private MigrateModuleProperties underTest = new MigrateModuleProperties(db.database(), system2); + + @Before + public void setup() { + String projectUuid = uuidFactory.create(); + String moduleUuid = uuidFactory.create(); + String subModule1Uuid = uuidFactory.create(); + String subModule2Uuid = uuidFactory.create(); + insertComponent(1, projectUuid, null, projectUuid, Qualifiers.PROJECT, "Multi-module project"); + insertComponent(2, moduleUuid, projectUuid, projectUuid, Qualifiers.MODULE, "Module"); + insertComponent(3, subModule1Uuid, moduleUuid, projectUuid, Qualifiers.MODULE, "Submodule 1"); + insertComponent(4, subModule2Uuid, moduleUuid, projectUuid, Qualifiers.MODULE, "Submodule 2"); + insertProperty(1, 1, "sonar.coverage.exclusions", "Proj1.java"); + insertProperty(2, 1, "sonar.cpd.exclusions", "Proj2.java"); + insertProperty(3, 2, "sonar.coverage.exclusions", "ModuleA.java"); + insertProperty(4, 2, "sonar.cpd.exclusions", "ModuleB.java"); + insertProperty(5, 3, "sonar.coverage.exclusions", "Module1A.java"); + insertProperty(6, 3, "sonar.cpd.exclusions", "Moddule1B.java"); + insertProperty(7, 4, "sonar.coverage.exclusions", "Module2A.java"); + insertProperty(8, 4, "sonar.cpd.exclusions", "Module2B.java"); + + String project2Uuid = uuidFactory.create(); + insertComponent(5, project2Uuid, null, project2Uuid, Qualifiers.PROJECT, "Single module project"); + insertProperty(9, 5, "sonar.coverage.exclusions", "SingleModuleA.java"); + insertProperty(10, 5, "sonar.cp.exclusions", "SingleModuleB.java"); + + String project3Uuid = uuidFactory.create(); + String singleModuleUuid = uuidFactory.create(); + insertComponent(6, project3Uuid, null, project3Uuid, Qualifiers.PROJECT, "Another multi-module project"); + insertComponent(7, singleModuleUuid, project3Uuid, project3Uuid, Qualifiers.MODULE, "Module X"); + insertProperty(11, 6, "sonar.coverage.exclusions", "InRoot.java"); + insertProperty(12, 7, "sonar.coverage.exclusions", "InModule.java"); + } + + @Test + public void migration() throws SQLException { + underTest.execute(); + + checkMigration(); + } + + @Test + public void migration_is_reentrant() throws SQLException { + underTest.execute(); + underTest.execute(); + + checkMigration(); + } + + private void checkMigration() { + final ImmutableSet expectedRemainingPropertyResourceIds = ImmutableSet.of(1L, 5L, 6L); + final String expectedProject1Property = "# previous settings for sub-project Multi-module project::Module\n" + + "sonar.coverage.exclusions=ModuleA.java\n" + + "sonar.cpd.exclusions=ModuleB.java\n" + + "\n" + + "# previous settings for sub-project Multi-module project::Submodule 1\n" + + "sonar.coverage.exclusions=Module1A.java\n" + + "sonar.cpd.exclusions=Moddule1B.java\n" + + "\n" + + "# previous settings for sub-project Multi-module project::Submodule 2\n" + + "sonar.coverage.exclusions=Module2A.java\n" + + "sonar.cpd.exclusions=Module2B.java\n"; + final String expectedProject2Property = "# previous settings for sub-project Another multi-module project::Module X\n" + + "sonar.coverage.exclusions=InModule.java\n"; + + List> newProperties = db.select("select ID, TEXT_VALUE, CLOB_VALUE " + + "from properties " + + "where PROP_KEY='sonar.subprojects.settings.removed' " + + "order by RESOURCE_ID"); + List> remainingProperties = db.select("select ID from properties"); + List> remainingPropertyResourceIds = db.select("select distinct RESOURCE_ID from properties"); + + assertThat(newProperties).hasSize(2); + Map project1Property = newProperties.get(0); + assertThat(project1Property.get("TEXT_VALUE")).isNull(); + assertThat(project1Property.get("CLOB_VALUE")).isEqualTo(expectedProject1Property); + Map project2Property = newProperties.get(1); + assertThat(project2Property.get("TEXT_VALUE")).isNull(); + assertThat(project2Property.get("CLOB_VALUE")).isEqualTo(expectedProject2Property); + assertThat(remainingProperties).hasSize(7); + assertThat(remainingPropertyResourceIds).hasSize(3); + assertThat(remainingPropertyResourceIds).allMatch(entry -> expectedRemainingPropertyResourceIds.contains(entry.get("RESOURCE_ID"))); + } + + private void insertComponent(long id, String uuid, @Nullable String rootUuid, String projectUuid, String qualifier, String name) { + db.executeInsert( + "projects", + "ID", valueOf(id), + "UUID", uuid, + "ROOT_UUID", rootUuid, + "PROJECT_UUID", projectUuid, + "SCOPE", Scopes.PROJECT, + "QUALIFIER", qualifier, + "NAME", name); + } + + private void insertProperty(long id, long componentId, String key, String value) { + db.executeInsert( + "properties", + "ID", valueOf(id), + "RESOURCE_ID", componentId, + "PROP_KEY", key, + "TEXT_VALUE", value, + "IS_EMPTY", false); + } +} diff --git a/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v75/MigrateModulePropertiesTest/schema.sql b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v75/MigrateModulePropertiesTest/schema.sql new file mode 100644 index 00000000000..1cbb2b7781e --- /dev/null +++ b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v75/MigrateModulePropertiesTest/schema.sql @@ -0,0 +1,35 @@ +CREATE TABLE "PROJECTS" ( + "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "KEE" VARCHAR(400), + "UUID" VARCHAR(50) NOT NULL, + "ROOT_UUID" VARCHAR(50), + "PROJECT_UUID" VARCHAR(50) NOT NULL, + "MODULE_UUID" VARCHAR(50), + "MODULE_UUID_PATH" VARCHAR(1500), + "MAIN_BRANCH_PROJECT_UUID" VARCHAR(50), + "NAME" VARCHAR(2000), + "TAGS" VARCHAR(500), + "ENABLED" BOOLEAN NOT NULL DEFAULT TRUE, + "SCOPE" VARCHAR(3), + "QUALIFIER" VARCHAR(10) +); +CREATE UNIQUE INDEX "PROJECTS_KEE" ON "PROJECTS" ("KEE"); +CREATE INDEX "PROJECTS_ROOT_UUID" ON "PROJECTS" ("ROOT_UUID"); +CREATE UNIQUE INDEX "PROJECTS_UUID" ON "PROJECTS" ("UUID"); +CREATE INDEX "PROJECTS_PROJECT_UUID" ON "PROJECTS" ("PROJECT_UUID"); +CREATE INDEX "PROJECTS_MODULE_UUID" ON "PROJECTS" ("MODULE_UUID"); +CREATE INDEX "PROJECTS_QUALIFIER" ON "PROJECTS" ("QUALIFIER"); + +CREATE TABLE "PROPERTIES" ( + "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "PROP_KEY" VARCHAR(512) NOT NULL, + "RESOURCE_ID" INTEGER, + "USER_ID" INTEGER, + "TEXT_VALUE" VARCHAR(4000), + "CLOB_VALUE" CLOB(2147483647), + "CREATED_AT" BIGINT, + "IS_EMPTY" BOOLEAN NOT NULL, +); +CREATE INDEX "PROPERTIES_KEY" ON "PROPERTIES" ("PROP_KEY"); + + -- 2.39.5