From ce00914d138c4d79a0a3bdb9e23504e6e4cfe9a7 Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Thu, 9 Jul 2015 12:16:21 +0200 Subject: [PATCH] SONAR-6702 Add migration to remove duplicated components with same key --- .../db/migrations/MigrationStepModule.java | 5 +- .../v52/RemoveDuplicatedComponentKeys.java | 163 ++++++++++++++++++ .../RemoveDuplicatedComponentKeysTest.java | 78 +++++++++ .../keep_enable_component-result.xml | 18 ++ .../keep_enable_component.xml | 26 +++ .../keep_last_component-result.xml | 18 ++ .../keep_last_component.xml | 26 +++ .../migrate-result.xml | 24 +++ .../migrate.xml | 30 ++++ .../schema.sql | 17 ++ .../925_remove_duplicated_component_keys.rb | 31 ++++ .../org/sonar/db/version/DatabaseVersion.java | 2 +- .../org/sonar/db/version/rows-h2.sql | 1 + 13 files changed, 437 insertions(+), 2 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeys.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest.java create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_enable_component-result.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_enable_component.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_last_component-result.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_last_component.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/migrate-result.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/migrate.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/schema.sql create mode 100644 server/sonar-web/src/main/webapp/WEB-INF/db/migrate/925_remove_duplicated_component_keys.rb diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MigrationStepModule.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MigrationStepModule.java index 398053e5d44..d565bc6689d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MigrationStepModule.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MigrationStepModule.java @@ -71,6 +71,7 @@ import org.sonar.server.db.migrations.v52.FeedMetricsBooleans; import org.sonar.server.db.migrations.v52.FeedProjectLinksComponentUuid; import org.sonar.server.db.migrations.v52.MoveProjectProfileAssociation; import org.sonar.server.db.migrations.v52.RemoveComponentLibraries; +import org.sonar.server.db.migrations.v52.RemoveDuplicatedComponentKeys; import org.sonar.server.db.migrations.v52.RemoveSnapshotLibraries; public class MigrationStepModule extends Module { @@ -144,6 +145,8 @@ public class MigrationStepModule extends Module { AddManualMeasuresComponentUuidColumn.class, FeedManualMeasuresComponentUuid.class, RemoveSnapshotLibraries.class, - RemoveComponentLibraries.class); + RemoveComponentLibraries.class, + RemoveDuplicatedComponentKeys.class + ); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeys.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeys.java new file mode 100644 index 00000000000..724b6076b9c --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeys.java @@ -0,0 +1,163 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.db.migrations.v52; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.FluentIterable; +import java.sql.SQLException; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nonnull; +import org.sonar.core.util.ProgressLogger; +import org.sonar.db.Database; +import org.sonar.server.db.migrations.BaseDataChange; +import org.sonar.server.db.migrations.Select; +import org.sonar.server.db.migrations.Upsert; + +/** + * Remove all duplicated component that have the same keys. + * For each duplicated component key : + * + */ +public class RemoveDuplicatedComponentKeys extends BaseDataChange { + + private final AtomicLong counter = new AtomicLong(0L); + + public RemoveDuplicatedComponentKeys(Database db) { + super(db); + } + + @Override + public void execute(final Context context) throws SQLException { + Upsert componentUpdate = context.prepareUpsert("DELETE FROM projects WHERE id=?"); + Upsert issuesUpdate = context.prepareUpsert("UPDATE issues SET component_uuid=?, project_uuid=? WHERE component_uuid=?"); + + ProgressLogger progress = ProgressLogger.create(getClass(), counter); + progress.start(); + try { + RemoveDuplicatedComponentHandler handler = new RemoveDuplicatedComponentHandler(context, componentUpdate, issuesUpdate); + context.prepareSelect( + "SELECT p.kee, COUNT(p.kee) FROM projects p " + + "GROUP BY p.kee " + + "HAVING COUNT(p.kee) > 1") + .scroll(handler); + if (!handler.isEmpty) { + componentUpdate.execute().commit(); + issuesUpdate.execute().commit(); + } + progress.log(); + } finally { + progress.stop(); + componentUpdate.close(); + issuesUpdate.close(); + } + } + + private class RemoveDuplicatedComponentHandler implements Select.RowHandler { + private final Context context; + private final Upsert componentUpdate; + private final Upsert issuesUpdate; + + private boolean isEmpty = true; + + public RemoveDuplicatedComponentHandler(Context context, Upsert componentUpdate, Upsert issuesUpdate) { + this.context = context; + this.componentUpdate = componentUpdate; + this.issuesUpdate = issuesUpdate; + } + + @Override + public void handle(Select.Row row) throws SQLException { + List components = context + .prepareSelect("SELECT p.id, p.uuid, p.project_uuid, p.enabled FROM projects p WHERE p.kee=? ORDER BY id") + .setString(1, row.getString(1)) + .list(ComponentRowReader.INSTANCE); + // We keep the enabled component or the last component of the list + Component refComponent = FluentIterable.from(components).firstMatch(EnabledComponent.INSTANCE).or(components.get(components.size() - 1)); + for (Component componentToRemove : FluentIterable.from(components).filter(Predicates.not(new MatchComponentId(refComponent.id)))) { + componentUpdate + .setLong(1, componentToRemove.id) + .addBatch(); + issuesUpdate + .setString(1, refComponent.uuid) + .setString(2, refComponent.projectUuid) + .setString(3, componentToRemove.uuid) + .addBatch(); + counter.getAndIncrement(); + isEmpty = false; + } + } + + public boolean isEmpty() { + return isEmpty; + } + } + + private enum EnabledComponent implements Predicate { + INSTANCE; + + @Override + public boolean apply(@Nonnull Component input) { + return input.enabled; + } + } + + private static class MatchComponentId implements Predicate { + + private final long id; + + public MatchComponentId(long id) { + this.id = id; + } + + @Override + public boolean apply(@Nonnull Component input) { + return input.id == this.id; + } + } + + private enum ComponentRowReader implements Select.RowReader { + INSTANCE; + + @Override + public Component read(Select.Row row) throws SQLException { + return new Component(row.getLong(1), row.getString(2), row.getString(3), row.getBoolean(4)); + } + } + + private static class Component { + private final long id; + private final String uuid; + private final String projectUuid; + private final boolean enabled; + + public Component(long id, String uuid, String projectUuid, boolean enabled) { + this.id = id; + this.uuid = uuid; + this.projectUuid = projectUuid; + this.enabled = enabled; + } + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest.java b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest.java new file mode 100644 index 00000000000..718723f82a5 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest.java @@ -0,0 +1,78 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.db.migrations.v52; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.db.DbTester; +import org.sonar.server.db.migrations.MigrationStep; + +public class RemoveDuplicatedComponentKeysTest { + + @ClassRule + public static DbTester db = DbTester.createForSchema(System2.INSTANCE, RemoveDuplicatedComponentKeysTest.class, "schema.sql"); + + MigrationStep migration; + + @Before + public void setUp() { + db.executeUpdateSql("truncate table projects"); + db.executeUpdateSql("truncate table issues"); + + migration = new RemoveDuplicatedComponentKeys(db.database()); + } + + @Test + public void migrate_empty_db() throws Exception { + migration.execute(); + } + + @Test + public void migrate_components_and_issues() throws Exception { + db.prepareDbUnit(this.getClass(), "migrate.xml"); + migration.execute(); + db.assertDbUnit(this.getClass(), "migrate-result.xml", "projects", "issues"); + } + + @Test + public void not_migrate_components_and_issues_already_migrated() throws Exception { + db.prepareDbUnit(this.getClass(), "migrate-result.xml"); + migration.execute(); + db.assertDbUnit(this.getClass(), "migrate-result.xml", "projects", "issues"); + } + + @Test + public void keep_enable_component_when_enabled_component_exists() throws Exception { + db.prepareDbUnit(this.getClass(), "keep_enable_component.xml"); + migration.execute(); + db.assertDbUnit(this.getClass(), "keep_enable_component-result.xml", "projects"); + } + + @Test + public void keep_last_component_when_no_enabled_components() throws Exception { + db.prepareDbUnit(this.getClass(), "keep_last_component.xml"); + migration.execute(); + db.assertDbUnit(this.getClass(), "keep_last_component-result.xml", "projects"); + } + +} diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_enable_component-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_enable_component-result.xml new file mode 100644 index 00000000000..7daf13a7aa6 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_enable_component-result.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_enable_component.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_enable_component.xml new file mode 100644 index 00000000000..9d04e8ae4d4 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_enable_component.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_last_component-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_last_component-result.xml new file mode 100644 index 00000000000..1fd89af0889 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_last_component-result.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_last_component.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_last_component.xml new file mode 100644 index 00000000000..45e6d0acc02 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/keep_last_component.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/migrate-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/migrate-result.xml new file mode 100644 index 00000000000..9b5eb5e857b --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/migrate-result.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/migrate.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/migrate.xml new file mode 100644 index 00000000000..84d3de065fc --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/migrate.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/schema.sql b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/schema.sql new file mode 100644 index 00000000000..19915dfd3b0 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v52/RemoveDuplicatedComponentKeysTest/schema.sql @@ -0,0 +1,17 @@ +CREATE TABLE "PROJECTS" ( + "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "KEE" VARCHAR(400), + "UUID" VARCHAR(50), + "PROJECT_UUID" VARCHAR(50), + "ENABLED" BOOLEAN NOT NULL DEFAULT TRUE +); + +CREATE INDEX "PROJECTS_KEE" ON "PROJECTS" ("KEE", "ENABLED"); + +CREATE TABLE "ISSUES" ( + "ID" BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "COMPONENT_UUID" VARCHAR(50), + "PROJECT_UUID" VARCHAR(50) +); + +CREATE INDEX "ISSUES_COMPONENT_UUID" ON "ISSUES" ("COMPONENT_UUID"); diff --git a/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/925_remove_duplicated_component_keys.rb b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/925_remove_duplicated_component_keys.rb new file mode 100644 index 00000000000..9587a59029e --- /dev/null +++ b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/925_remove_duplicated_component_keys.rb @@ -0,0 +1,31 @@ +# +# SonarQube, open source software quality management tool. +# Copyright (C) 2008-2014 SonarSource +# mailto:contact AT sonarsource DOT com +# +# SonarQube is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# SonarQube is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +# +# SonarQube 5.2 +# SONAR-6702 +# +class RemoveDuplicatedComponentKeys < ActiveRecord::Migration + + def self.up + execute_java_migration('org.sonar.server.db.migrations.v52.RemoveDuplicatedComponentKeys') + end + +end diff --git a/sonar-db/src/main/java/org/sonar/db/version/DatabaseVersion.java b/sonar-db/src/main/java/org/sonar/db/version/DatabaseVersion.java index 7313d12d2f8..6203ef6c969 100644 --- a/sonar-db/src/main/java/org/sonar/db/version/DatabaseVersion.java +++ b/sonar-db/src/main/java/org/sonar/db/version/DatabaseVersion.java @@ -28,7 +28,7 @@ import org.sonar.db.MyBatis; public class DatabaseVersion { - public static final int LAST_VERSION = 924; + public static final int LAST_VERSION = 925; /** * List of all the tables. diff --git a/sonar-db/src/main/resources/org/sonar/db/version/rows-h2.sql b/sonar-db/src/main/resources/org/sonar/db/version/rows-h2.sql index dd4afdca565..007a5bfe883 100644 --- a/sonar-db/src/main/resources/org/sonar/db/version/rows-h2.sql +++ b/sonar-db/src/main/resources/org/sonar/db/version/rows-h2.sql @@ -343,6 +343,7 @@ INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('921'); INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('922'); INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('923'); INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('924'); +INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('925'); INSERT INTO USERS(ID, LOGIN, NAME, EMAIL, CRYPTED_PASSWORD, SALT, CREATED_AT, UPDATED_AT, REMEMBER_TOKEN, REMEMBER_TOKEN_EXPIRES_AT) VALUES (1, 'admin', 'Administrator', '', 'a373a0e667abb2604c1fd571eb4ad47fe8cc0878', '48bc4b0d93179b5103fd3885ea9119498e9d161b', '1418215735482', '1418215735482', null, null); ALTER TABLE USERS ALTER COLUMN ID RESTART WITH 2; -- 2.39.5