From 884491773414860046bd23cea9fff675a0a95a3c Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Tue, 20 Oct 2020 17:29:02 +0200 Subject: [PATCH] SONAR-13913 Remove tables and indexes clean up used in Integration Tests * SONAR-13913 Remove usage of Orchestrator#resetData * Remove ProjectAnalysisRule * Upgrade to Orchestrator 3.31 * SONAR-13913 Remove BackendCleanup class * Remove code related to no more existing tables --- build.gradle | 2 +- .../java/org/sonar/db/version/SqTables.java | 17 -- .../db/migration/charset/ColumnDef.java | 2 +- .../db/migration/charset/ColumnDefTest.java | 5 - .../sonar/server/platform/BackendCleanup.java | 251 ------------------ .../server/platform/BackendCleanupTest.java | 159 ----------- .../platformlevel/PlatformLevel4.java | 2 - 7 files changed, 2 insertions(+), 436 deletions(-) delete mode 100644 server/sonar-webserver-core/src/main/java/org/sonar/server/platform/BackendCleanup.java delete mode 100644 server/sonar-webserver-core/src/test/java/org/sonar/server/platform/BackendCleanupTest.java diff --git a/build.gradle b/build.gradle index fac0ccfecd8..1bd7a00ba8d 100644 --- a/build.gradle +++ b/build.gradle @@ -370,7 +370,7 @@ subprojects { dependency 'org.postgresql:postgresql:42.2.17' dependency 'org.reflections:reflections:0.9.12' dependency 'org.simpleframework:simple:4.1.21' - dependency 'org.sonarsource.orchestrator:sonar-orchestrator:3.30.0.2630' + dependency 'org.sonarsource.orchestrator:sonar-orchestrator:3.31.0.2646' dependency 'org.sonarsource.update-center:sonar-update-center-common:1.23.0.723' dependency 'org.subethamail:subethasmtp:3.1.7' dependency 'org.yaml:snakeyaml:1.26' diff --git a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java index 4c2f24626c0..4c705ec451d 100644 --- a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java +++ b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java @@ -26,23 +26,6 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableSet; public final class SqTables { - /** - * These tables are still involved in DB migrations, so potentially - * incorrect collation must be fixed so that joins with other - * tables are possible. - */ - public static final Set OLD_DROPPED_TABLES = unmodifiableSet(new HashSet<>(asList( - "active_dashboards", - "activities", - "dashboards", - "issue_filters", - "issue_filter_favourites", - "loaded_templates", - "measure_filters", - "measure_filter_favourites", - "resource_index", - "widgets", - "widget_properties"))); /** * List of all the tables. diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java index a0695f79fc8..5666d5cf938 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java @@ -81,7 +81,7 @@ public class ColumnDef { public boolean isInSonarQubeTable() { String tableName = table.toLowerCase(Locale.ENGLISH); - return SqTables.TABLES.contains(tableName) || SqTables.OLD_DROPPED_TABLES.contains(tableName); + return SqTables.TABLES.contains(tableName); } public enum ColumnDefRowConverter implements SqlExecutor.RowConverter { diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/charset/ColumnDefTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/charset/ColumnDefTest.java index c9ab3f9d4a0..f1c50ff4866 100644 --- a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/charset/ColumnDefTest.java +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/charset/ColumnDefTest.java @@ -43,9 +43,4 @@ public class ColumnDefTest { assertThat(underTest.isInSonarQubeTable()).isTrue(); } - @Test - public void isInSonarQubeTable_returns_true_if_table_existed_in_previous_versions_of_sonarqube() { - ColumnDef underTest = new ColumnDef("activities", "kee", "charset", "collation", "NVARCHAR", 100L, false); - assertThat(underTest.isInSonarQubeTable()).isTrue(); - } } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/BackendCleanup.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/BackendCleanup.java deleted file mode 100644 index b7bbd9b0782..00000000000 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/BackendCleanup.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.platform; - -import com.google.common.collect.ImmutableMap; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import org.sonar.api.server.ServerSide; -import org.sonar.api.utils.log.Loggers; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.dialect.Oracle; -import org.sonar.db.version.SqTables; -import org.sonar.server.component.index.ComponentIndexDefinition; -import org.sonar.server.es.BulkIndexer; -import org.sonar.server.es.EsClient; -import org.sonar.server.es.Index; -import org.sonar.server.es.IndexType; -import org.sonar.server.issue.index.IssueIndexDefinition; -import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition; -import org.sonar.server.property.InternalProperties; -import org.sonar.server.view.index.ViewIndexDefinition; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; - -@ServerSide -public class BackendCleanup { - - private static final String[] ANALYSIS_TABLES = { - "ce_activity", "ce_queue", "ce_task_input", "ce_scanner_context", - "duplications_index", "events", "issues", "issue_changes", "manual_measures", - "notifications", "project_links", "project_measures", "components", "projects", - "snapshots", "file_sources", "webhook_deliveries" - }; - private static final String[] COMPONENT_RELATED_TABLES = { - "group_roles", "user_roles", "properties" - }; - private static final Map TABLE_CLEANERS = ImmutableMap.of( - "organizations", BackendCleanup::truncateOrganizations, - "users", BackendCleanup::truncateUsers, - "groups", BackendCleanup::truncateGroups, - "internal_properties", BackendCleanup::truncateInternalProperties, - "schema_migrations", BackendCleanup::truncateSchemaMigrations); - - private final EsClient esClient; - private final DbClient dbClient; - - public BackendCleanup(EsClient esClient, DbClient dbClient) { - this.esClient = esClient; - this.dbClient = dbClient; - } - - public void clearAll() { - clearDb(); - clearIndexes(); - } - - public void clearDb() { - try (DbSession dbSession = dbClient.openSession(false); - Connection connection = dbSession.getConnection(); - Statement ddlStatement = connection.createStatement()) { - for (String tableName : SqTables.TABLES) { - Optional.ofNullable(TABLE_CLEANERS.get(tableName)) - .orElse(BackendCleanup::truncateDefault) - .clean(tableName, ddlStatement, connection); - } - } catch (Exception e) { - throw new IllegalStateException("Fail to clear db", e); - } - } - - public void clearIndexes() { - Loggers.get(getClass()).info("Truncate Elasticsearch indices"); - try { - esClient.prepareClearCache().get(); - - for (String index : esClient.prepareState().get().getState().getMetaData().getConcreteAllIndices()) { - /*under the hood, type is not used to perform clearIndex, so it's ok it does not match any existing index*/ - clearIndex(Index.simple(index)); - } - } catch (Exception e) { - throw new IllegalStateException("Unable to clear indexes", e); - } - } - - /** - * Reset data in order to to be in same state as a fresh installation (but without having to drop db and restart the server). - * - * Please be careful when updating this method as it's called by Orchestrator. - */ - public void resetData() { - try (DbSession dbSession = dbClient.openSession(false); - Connection connection = dbSession.getConnection()) { - - truncateAnalysisTables(connection); - deleteManualRules(connection); - truncateInternalProperties(null, null, connection); - truncateUsers(null, null, connection); - truncateOrganizations(null, null, connection); - } catch (SQLException e) { - throw new IllegalStateException("Fail to reset data", e); - } - - clearIndex(IssueIndexDefinition.DESCRIPTOR); - clearIndex(ViewIndexDefinition.DESCRIPTOR); - clearIndex(ProjectMeasuresIndexDefinition.DESCRIPTOR); - clearIndex(ComponentIndexDefinition.DESCRIPTOR); - } - - private void truncateAnalysisTables(Connection connection) throws SQLException { - try (Statement statement = connection.createStatement()) { - // Clear inspection tables - for (String table : ANALYSIS_TABLES) { - statement.execute(createTruncateSql(table.toLowerCase(Locale.ENGLISH))); - // commit is useless on some databases - connection.commit(); - } - for (String table : COMPONENT_RELATED_TABLES) { - statement.execute("DELETE FROM " + table + " WHERE component_uuid IS NOT NULL"); - connection.commit(); - } - } - } - - private String createTruncateSql(String table) { - if (dbClient.getDatabase().getDialect().getId().equals(Oracle.ID)) { - // truncate operation is needs to lock the table on Oracle. Unfortunately - // it fails sometimes in our QA environment because table is locked. - // We never found the root cause (no locks found when displaying them just after - // receiving the error). - // Workaround is to use "delete" operation. It does not require lock on table. - return "DELETE FROM " + table; - } - return "TRUNCATE TABLE " + table; - } - - private static void deleteManualRules(Connection connection) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement("DELETE FROM rules WHERE rules.plugin_name='manual'")) { - statement.execute(); - // commit is useless on some databases - connection.commit(); - } - } - - /** - * Completely remove a index with all types - */ - public void clearIndex(Index index) { - BulkIndexer.delete(esClient, IndexType.main(index, index.getName()), esClient.prepareSearch(index).setQuery(matchAllQuery())); - } - - @FunctionalInterface - private interface TableCleaner { - void clean(String tableName, Statement ddlStatement, Connection connection) throws SQLException; - } - - private static void truncateDefault(String tableName, Statement ddlStatement, Connection connection) throws SQLException { - ddlStatement.execute("TRUNCATE TABLE " + tableName.toLowerCase(Locale.ENGLISH)); - // commit is useless on some databases - connection.commit(); - } - - /** - * Default organization must never be deleted - */ - private static void truncateOrganizations(String tableName, Statement ddlStatement, Connection connection) throws SQLException { - try (PreparedStatement preparedStatement = connection.prepareStatement("delete from organizations where kee <> ?")) { - preparedStatement.setString(1, "default-organization"); - preparedStatement.execute(); - // commit is useless on some databases - connection.commit(); - } - } - - /** - * User admin must never be deleted. - */ - private static void truncateUsers(String tableName, Statement ddlStatement, Connection connection) throws SQLException { - try (PreparedStatement preparedStatement = connection.prepareStatement("delete from users where login <> ?")) { - preparedStatement.setString(1, "admin"); - preparedStatement.execute(); - // commit is useless on some databases - connection.commit(); - } - // "admin" is not flagged as root by default - try (PreparedStatement preparedStatement = connection.prepareStatement("update users set is_root=?")) { - preparedStatement.setBoolean(1, false); - preparedStatement.execute(); - // commit is useless on some databases - connection.commit(); - } - } - - /** - * Groups sonar-users is referenced by the default organization as its default group. - */ - private static void truncateGroups(String tableName, Statement ddlStatement, Connection connection) throws SQLException { - try (PreparedStatement preparedStatement = connection.prepareStatement("delete from groups where name <> ?")) { - preparedStatement.setString(1, "sonar-users"); - preparedStatement.execute(); - // commit is useless on some databases - connection.commit(); - } - } - - /** - * Internal property {@link InternalProperties#DEFAULT_ORGANIZATION} must never be deleted. - */ - private static void truncateInternalProperties(String tableName, Statement ddlStatement, Connection connection) throws SQLException { - try (PreparedStatement preparedStatement = connection.prepareStatement("delete from internal_properties where kee not in (?,?,?,?,?)")) { - preparedStatement.setString(1, InternalProperties.DEFAULT_ORGANIZATION); - preparedStatement.setString(2, InternalProperties.SERVER_ID_CHECKSUM); - preparedStatement.setString(3, InternalProperties.DEFAULT_PROJECT_TEMPLATE); - preparedStatement.setString(4, InternalProperties.DEFAULT_PORTFOLIO_TEMPLATE); - preparedStatement.setString(5, InternalProperties.DEFAULT_APPLICATION_TEMPLATE); - preparedStatement.execute(); - // commit is useless on some databases - connection.commit(); - } - } - - /** - * Data in SCHEMA_MIGRATIONS table is inserted when DB is created and should not be altered afterwards. - */ - private static void truncateSchemaMigrations(String tableName, Statement ddlStatement, Connection connection) { - // do nothing - } - -} diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/BackendCleanupTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/BackendCleanupTest.java deleted file mode 100644 index dd8173b2367..00000000000 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/BackendCleanupTest.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.platform; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.System2; -import org.sonar.core.util.Uuids; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.ComponentTesting; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.organization.OrganizationTesting; -import org.sonar.db.property.PropertyDto; -import org.sonar.db.rule.RuleTesting; -import org.sonar.server.component.index.ComponentDoc; -import org.sonar.server.component.index.ComponentIndexDefinition; -import org.sonar.server.component.index.ComponentIndexer; -import org.sonar.server.es.EsTester; -import org.sonar.server.issue.IssueDocTesting; -import org.sonar.server.issue.index.IssueIndexDefinition; -import org.sonar.server.measure.index.ProjectMeasuresDoc; -import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition; -import org.sonar.server.rule.index.RuleDoc; -import org.sonar.server.rule.index.RuleIndexDefinition; -import org.sonar.server.view.index.ViewDoc; -import org.sonar.server.view.index.ViewIndexDefinition; - -import static com.google.common.collect.Lists.newArrayList; -import static org.assertj.core.api.Assertions.assertThat; - -public class BackendCleanupTest { - - @Rule - public EsTester es = EsTester.create(); - - @Rule - public DbTester dbTester = DbTester.create(System2.INSTANCE); - - private BackendCleanup underTest = new BackendCleanup(es.client(), dbTester.getDbClient()); - private OrganizationDto organization; - - @Before - public void setUp() { - organization = OrganizationTesting.newOrganizationDto(); - } - - @Test - public void clear_db() { - insertSomeData(); - - underTest.clearDb(); - - assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0); - assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0); - assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0); - assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0); - assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0); - } - - @Test - public void clear_indexes() { - es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc()); - es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc()); - es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc()); - - underTest.clearIndexes(); - - assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0); - assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0); - } - - @Test - public void clear_all() { - insertSomeData(); - - es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc()); - es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc()); - es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc()); - - underTest.clearAll(); - - assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0); - assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(0); - assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0); - - assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0); - assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0); - assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0); - assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0); - assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0); - } - - @Test - public void reset_data() { - insertSomeData(); - - es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc()); - es.putDocuments(ViewIndexDefinition.TYPE_VIEW, new ViewDoc().setUuid("CDEF").setProjects(newArrayList("DEFG"))); - es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc()); - es.putDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES, new ProjectMeasuresDoc() - .setId("PROJECT") - .setKey("Key") - .setName("Name")); - es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc()); - - underTest.resetData(); - - assertThat(dbTester.countRowsOfTable("projects")).isZero(); - assertThat(dbTester.countRowsOfTable("components")).isZero(); - assertThat(dbTester.countRowsOfTable("snapshots")).isZero(); - assertThat(dbTester.countRowsOfTable("properties")).isZero(); - assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isZero(); - assertThat(es.countDocuments(ViewIndexDefinition.TYPE_VIEW)).isZero(); - assertThat(es.countDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES)).isZero(); - assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isZero(); - - // Rules should not be removed - assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(1); - assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(1); - } - - private void insertSomeData() { - OrganizationDto organization = dbTester.organizations().insert(); - ComponentDto project = dbTester.components().insertPrivateProject(organization); - dbTester.components().insertSnapshot(project); - dbTester.rules().insert(); - dbTester.properties().insertProperty(new PropertyDto() - .setKey("sonar.profile.java") - .setValue("Sonar Way") - .setComponentUuid(project.uuid())); - } - - private static RuleDoc newRuleDoc() { - return new RuleDoc().setUuid(Uuids.createFast()).setKey(RuleTesting.XOO_X1.toString()).setRepository(RuleTesting.XOO_X1.repository()); - } - - private ComponentDoc newComponentDoc() { - return ComponentIndexer.toDocument(ComponentTesting.newPrivateProjectDto(organization)); - } -} diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index f1f0b4d26a3..23217a8527d 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -119,7 +119,6 @@ import org.sonar.server.permission.PermissionUpdater; import org.sonar.server.permission.UserPermissionChanger; import org.sonar.server.permission.index.PermissionIndexer; import org.sonar.server.permission.ws.PermissionsWsModule; -import org.sonar.server.platform.BackendCleanup; import org.sonar.server.platform.ClusterVerification; import org.sonar.server.platform.PersistentSettings; import org.sonar.server.platform.SystemInfoWriterModule; @@ -256,7 +255,6 @@ public class PlatformLevel4 extends PlatformLevel { DefaultResourceTypes.get(), SettingsChangeNotifier.class, ServerWs.class, - BackendCleanup.class, IndexDefinitions.class, WebAnalyticsLoaderImpl.class, -- 2.39.5