Revert "SONAR-13913 Remove tables and indexes clean up used in Integration Tests"

This reverts commit 6d13dd2f

build.gradle

@@ -370,7 +370,7 @@ subprojects {
       dependency 'org.postgresql:postgresql:42.2.17'
       dependency 'org.reflections:reflections:0.9.12'
       dependency 'org.simpleframework:simple:4.1.21'
-      dependency 'org.sonarsource.orchestrator:sonar-orchestrator:3.31.0.2646'
+      dependency 'org.sonarsource.orchestrator:sonar-orchestrator:3.30.0.2630'
       dependency 'org.sonarsource.update-center:sonar-update-center-common:1.23.0.723'
       dependency 'org.subethamail:subethasmtp:3.1.7'
       dependency 'org.yaml:snakeyaml:1.26'

server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java

@@ -26,6 +26,23 @@ import static java.util.Arrays.asList;
 import static java.util.Collections.unmodifiableSet;
 
 public final class SqTables {
+  /**
+   * These tables are still involved in DB migrations, so potentially
+   * incorrect collation must be fixed so that joins with other
+   * tables are possible.
+   */
+  public static final Set<String> OLD_DROPPED_TABLES = unmodifiableSet(new HashSet<>(asList(
+    "active_dashboards",
+    "activities",
+    "dashboards",
+    "issue_filters",
+    "issue_filter_favourites",
+    "loaded_templates",
+    "measure_filters",
+    "measure_filter_favourites",
+    "resource_index",
+    "widgets",
+    "widget_properties")));
 
   /**
    * List of all the tables.

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java

@@ -81,7 +81,7 @@ public class ColumnDef {
 
   public boolean isInSonarQubeTable() {
     String tableName = table.toLowerCase(Locale.ENGLISH);
-    return SqTables.TABLES.contains(tableName);
+    return SqTables.TABLES.contains(tableName) || SqTables.OLD_DROPPED_TABLES.contains(tableName);
   }
 
   public enum ColumnDefRowConverter implements SqlExecutor.RowConverter<ColumnDef> {

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/charset/ColumnDefTest.java

@@ -43,4 +43,9 @@ public class ColumnDefTest {
     assertThat(underTest.isInSonarQubeTable()).isTrue();
   }
 
+  @Test
+  public void isInSonarQubeTable_returns_true_if_table_existed_in_previous_versions_of_sonarqube() {
+    ColumnDef underTest = new ColumnDef("activities", "kee", "charset", "collation", "NVARCHAR", 100L, false);
+    assertThat(underTest.isInSonarQubeTable()).isTrue();
+  }
 }

server/sonar-webserver-core/src/main/java/org/sonar/server/platform/BackendCleanup.java

@@ -0,0 +1,251 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform;
+
+import com.google.common.collect.ImmutableMap;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import org.sonar.api.server.ServerSide;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.dialect.Oracle;
+import org.sonar.db.version.SqTables;
+import org.sonar.server.component.index.ComponentIndexDefinition;
+import org.sonar.server.es.BulkIndexer;
+import org.sonar.server.es.EsClient;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.issue.index.IssueIndexDefinition;
+import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition;
+import org.sonar.server.property.InternalProperties;
+import org.sonar.server.view.index.ViewIndexDefinition;
+
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+
+@ServerSide
+public class BackendCleanup {
+
+  private static final String[] ANALYSIS_TABLES = {
+    "ce_activity", "ce_queue", "ce_task_input", "ce_scanner_context",
+    "duplications_index", "events", "issues", "issue_changes", "manual_measures",
+    "notifications", "project_links", "project_measures", "components", "projects",
+    "snapshots", "file_sources", "webhook_deliveries"
+  };
+  private static final String[] COMPONENT_RELATED_TABLES = {
+    "group_roles", "user_roles", "properties"
+  };
+  private static final Map<String, TableCleaner> TABLE_CLEANERS = ImmutableMap.of(
+    "organizations", BackendCleanup::truncateOrganizations,
+    "users", BackendCleanup::truncateUsers,
+    "groups", BackendCleanup::truncateGroups,
+    "internal_properties", BackendCleanup::truncateInternalProperties,
+    "schema_migrations", BackendCleanup::truncateSchemaMigrations);
+
+  private final EsClient esClient;
+  private final DbClient dbClient;
+
+  public BackendCleanup(EsClient esClient, DbClient dbClient) {
+    this.esClient = esClient;
+    this.dbClient = dbClient;
+  }
+
+  public void clearAll() {
+    clearDb();
+    clearIndexes();
+  }
+
+  public void clearDb() {
+    try (DbSession dbSession = dbClient.openSession(false);
+      Connection connection = dbSession.getConnection();
+      Statement ddlStatement = connection.createStatement()) {
+      for (String tableName : SqTables.TABLES) {
+        Optional.ofNullable(TABLE_CLEANERS.get(tableName))
+          .orElse(BackendCleanup::truncateDefault)
+          .clean(tableName, ddlStatement, connection);
+      }
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to clear db", e);
+    }
+  }
+
+  public void clearIndexes() {
+    Loggers.get(getClass()).info("Truncate Elasticsearch indices");
+    try {
+      esClient.prepareClearCache().get();
+
+      for (String index : esClient.prepareState().get().getState().getMetaData().getConcreteAllIndices()) {
+        /*under the hood, type is not used to perform clearIndex, so it's ok it does not match any existing index*/
+        clearIndex(Index.simple(index));
+      }
+    } catch (Exception e) {
+      throw new IllegalStateException("Unable to clear indexes", e);
+    }
+  }
+
+  /**
+   * Reset data in order to to be in same state as a fresh installation (but without having to drop db and restart the server).
+   *
+   * Please be careful when updating this method as it's called by Orchestrator.
+   */
+  public void resetData() {
+    try (DbSession dbSession = dbClient.openSession(false);
+      Connection connection = dbSession.getConnection()) {
+
+      truncateAnalysisTables(connection);
+      deleteManualRules(connection);
+      truncateInternalProperties(null, null, connection);
+      truncateUsers(null, null, connection);
+      truncateOrganizations(null, null, connection);
+    } catch (SQLException e) {
+      throw new IllegalStateException("Fail to reset data", e);
+    }
+
+    clearIndex(IssueIndexDefinition.DESCRIPTOR);
+    clearIndex(ViewIndexDefinition.DESCRIPTOR);
+    clearIndex(ProjectMeasuresIndexDefinition.DESCRIPTOR);
+    clearIndex(ComponentIndexDefinition.DESCRIPTOR);
+  }
+
+  private void truncateAnalysisTables(Connection connection) throws SQLException {
+    try (Statement statement = connection.createStatement()) {
+      // Clear inspection tables
+      for (String table : ANALYSIS_TABLES) {
+        statement.execute(createTruncateSql(table.toLowerCase(Locale.ENGLISH)));
+        // commit is useless on some databases
+        connection.commit();
+      }
+      for (String table : COMPONENT_RELATED_TABLES) {
+        statement.execute("DELETE FROM " + table + " WHERE component_uuid IS NOT NULL");
+        connection.commit();
+      }
+    }
+  }
+
+  private String createTruncateSql(String table) {
+    if (dbClient.getDatabase().getDialect().getId().equals(Oracle.ID)) {
+      // truncate operation is needs to lock the table on Oracle. Unfortunately
+      // it fails sometimes in our QA environment because table is locked.
+      // We never found the root cause (no locks found when displaying them just after
+      // receiving the error).
+      // Workaround is to use "delete" operation. It does not require lock on table.
+      return "DELETE FROM " + table;
+    }
+    return "TRUNCATE TABLE " + table;
+  }
+
+  private static void deleteManualRules(Connection connection) throws SQLException {
+    try (PreparedStatement statement = connection.prepareStatement("DELETE FROM rules WHERE rules.plugin_name='manual'")) {
+      statement.execute();
+      // commit is useless on some databases
+      connection.commit();
+    }
+  }
+
+  /**
+   * Completely remove a index with all types
+   */
+  public void clearIndex(Index index) {
+    BulkIndexer.delete(esClient, IndexType.main(index, index.getName()), esClient.prepareSearch(index).setQuery(matchAllQuery()));
+  }
+
+  @FunctionalInterface
+  private interface TableCleaner {
+    void clean(String tableName, Statement ddlStatement, Connection connection) throws SQLException;
+  }
+
+  private static void truncateDefault(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+    ddlStatement.execute("TRUNCATE TABLE " + tableName.toLowerCase(Locale.ENGLISH));
+    // commit is useless on some databases
+    connection.commit();
+  }
+
+  /**
+   * Default organization must never be deleted
+   */
+  private static void truncateOrganizations(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+    try (PreparedStatement preparedStatement = connection.prepareStatement("delete from organizations where kee <> ?")) {
+      preparedStatement.setString(1, "default-organization");
+      preparedStatement.execute();
+      // commit is useless on some databases
+      connection.commit();
+    }
+  }
+
+  /**
+   * User admin must never be deleted.
+   */
+  private static void truncateUsers(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+    try (PreparedStatement preparedStatement = connection.prepareStatement("delete from users where login <> ?")) {
+      preparedStatement.setString(1, "admin");
+      preparedStatement.execute();
+      // commit is useless on some databases
+      connection.commit();
+    }
+    // "admin" is not flagged as root by default
+    try (PreparedStatement preparedStatement = connection.prepareStatement("update users set is_root=?")) {
+      preparedStatement.setBoolean(1, false);
+      preparedStatement.execute();
+      // commit is useless on some databases
+      connection.commit();
+    }
+  }
+
+  /**
+   * Groups sonar-users is referenced by the default organization as its default group.
+   */
+  private static void truncateGroups(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+    try (PreparedStatement preparedStatement = connection.prepareStatement("delete from groups where name <> ?")) {
+      preparedStatement.setString(1, "sonar-users");
+      preparedStatement.execute();
+      // commit is useless on some databases
+      connection.commit();
+    }
+  }
+
+  /**
+   * Internal property {@link InternalProperties#DEFAULT_ORGANIZATION} must never be deleted.
+   */
+  private static void truncateInternalProperties(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+    try (PreparedStatement preparedStatement = connection.prepareStatement("delete from internal_properties where kee not in (?,?,?,?,?)")) {
+      preparedStatement.setString(1, InternalProperties.DEFAULT_ORGANIZATION);
+      preparedStatement.setString(2, InternalProperties.SERVER_ID_CHECKSUM);
+      preparedStatement.setString(3, InternalProperties.DEFAULT_PROJECT_TEMPLATE);
+      preparedStatement.setString(4, InternalProperties.DEFAULT_PORTFOLIO_TEMPLATE);
+      preparedStatement.setString(5, InternalProperties.DEFAULT_APPLICATION_TEMPLATE);
+      preparedStatement.execute();
+      // commit is useless on some databases
+      connection.commit();
+    }
+  }
+
+  /**
+   * Data in SCHEMA_MIGRATIONS table is inserted when DB is created and should not be altered afterwards.
+   */
+  private static void truncateSchemaMigrations(String tableName, Statement ddlStatement, Connection connection) {
+    // do nothing
+  }
+
+}

server/sonar-webserver-core/src/test/java/org/sonar/server/platform/BackendCleanupTest.java

@@ -0,0 +1,159 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.utils.System2;
+import org.sonar.core.util.Uuids;
+import org.sonar.db.DbTester;
+import org.sonar.db.component.ComponentDto;
+import org.sonar.db.component.ComponentTesting;
+import org.sonar.db.organization.OrganizationDto;
+import org.sonar.db.organization.OrganizationTesting;
+import org.sonar.db.property.PropertyDto;
+import org.sonar.db.rule.RuleTesting;
+import org.sonar.server.component.index.ComponentDoc;
+import org.sonar.server.component.index.ComponentIndexDefinition;
+import org.sonar.server.component.index.ComponentIndexer;
+import org.sonar.server.es.EsTester;
+import org.sonar.server.issue.IssueDocTesting;
+import org.sonar.server.issue.index.IssueIndexDefinition;
+import org.sonar.server.measure.index.ProjectMeasuresDoc;
+import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition;
+import org.sonar.server.rule.index.RuleDoc;
+import org.sonar.server.rule.index.RuleIndexDefinition;
+import org.sonar.server.view.index.ViewDoc;
+import org.sonar.server.view.index.ViewIndexDefinition;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class BackendCleanupTest {
+
+  @Rule
+  public EsTester es = EsTester.create();
+
+  @Rule
+  public DbTester dbTester = DbTester.create(System2.INSTANCE);
+
+  private BackendCleanup underTest = new BackendCleanup(es.client(), dbTester.getDbClient());
+  private OrganizationDto organization;
+
+  @Before
+  public void setUp() {
+    organization = OrganizationTesting.newOrganizationDto();
+  }
+
+  @Test
+  public void clear_db() {
+    insertSomeData();
+
+    underTest.clearDb();
+
+    assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0);
+    assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0);
+    assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0);
+    assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0);
+    assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0);
+  }
+
+  @Test
+  public void clear_indexes() {
+    es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
+    es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
+    es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
+
+    underTest.clearIndexes();
+
+    assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0);
+    assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0);
+  }
+
+  @Test
+  public void clear_all() {
+    insertSomeData();
+
+    es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
+    es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
+    es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
+
+    underTest.clearAll();
+
+    assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0);
+    assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(0);
+    assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0);
+
+    assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0);
+    assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0);
+    assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0);
+    assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0);
+    assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0);
+  }
+
+  @Test
+  public void reset_data() {
+    insertSomeData();
+
+    es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
+    es.putDocuments(ViewIndexDefinition.TYPE_VIEW, new ViewDoc().setUuid("CDEF").setProjects(newArrayList("DEFG")));
+    es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
+    es.putDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES, new ProjectMeasuresDoc()
+      .setId("PROJECT")
+      .setKey("Key")
+      .setName("Name"));
+    es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
+
+    underTest.resetData();
+
+    assertThat(dbTester.countRowsOfTable("projects")).isZero();
+    assertThat(dbTester.countRowsOfTable("components")).isZero();
+    assertThat(dbTester.countRowsOfTable("snapshots")).isZero();
+    assertThat(dbTester.countRowsOfTable("properties")).isZero();
+    assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isZero();
+    assertThat(es.countDocuments(ViewIndexDefinition.TYPE_VIEW)).isZero();
+    assertThat(es.countDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES)).isZero();
+    assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isZero();
+
+    // Rules should not be removed
+    assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(1);
+    assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(1);
+  }
+
+  private void insertSomeData() {
+    OrganizationDto organization = dbTester.organizations().insert();
+    ComponentDto project = dbTester.components().insertPrivateProject(organization);
+    dbTester.components().insertSnapshot(project);
+    dbTester.rules().insert();
+    dbTester.properties().insertProperty(new PropertyDto()
+      .setKey("sonar.profile.java")
+      .setValue("Sonar Way")
+      .setComponentUuid(project.uuid()));
+  }
+
+  private static RuleDoc newRuleDoc() {
+    return new RuleDoc().setUuid(Uuids.createFast()).setKey(RuleTesting.XOO_X1.toString()).setRepository(RuleTesting.XOO_X1.repository());
+  }
+
+  private ComponentDoc newComponentDoc() {
+    return ComponentIndexer.toDocument(ComponentTesting.newPrivateProjectDto(organization));
+  }
+}

server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java

@@ -119,6 +119,7 @@ import org.sonar.server.permission.PermissionUpdater;
 import org.sonar.server.permission.UserPermissionChanger;
 import org.sonar.server.permission.index.PermissionIndexer;
 import org.sonar.server.permission.ws.PermissionsWsModule;
+import org.sonar.server.platform.BackendCleanup;
 import org.sonar.server.platform.ClusterVerification;
 import org.sonar.server.platform.PersistentSettings;
 import org.sonar.server.platform.SystemInfoWriterModule;
@@ -255,6 +256,7 @@ public class PlatformLevel4 extends PlatformLevel {
       DefaultResourceTypes.get(),
       SettingsChangeNotifier.class,
       ServerWs.class,
+      BackendCleanup.class,
       IndexDefinitions.class,
       WebAnalyticsLoaderImpl.class,