SONAR-13913 Remove tables and indexes clean up used in Integration Tests

* SONAR-13913 Remove usage of Orchestrator#resetData

* Remove ProjectAnalysisRule

* Upgrade to Orchestrator 3.31

* SONAR-13913 Remove BackendCleanup class

* Remove code related to no more existing tables

7 files changed, 2 insertions, 436 deletions

diff --git a/build.gradle b/build.gradle
index fac0ccfecd8..1bd7a00ba8d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -370,7 +370,7 @@ subprojects {
       dependency 'org.postgresql:postgresql:42.2.17'
       dependency 'org.reflections:reflections:0.9.12'
       dependency 'org.simpleframework:simple:4.1.21'
-      dependency 'org.sonarsource.orchestrator:sonar-orchestrator:3.30.0.2630'
+      dependency 'org.sonarsource.orchestrator:sonar-orchestrator:3.31.0.2646'
       dependency 'org.sonarsource.update-center:sonar-update-center-common:1.23.0.723'
       dependency 'org.subethamail:subethasmtp:3.1.7'
       dependency 'org.yaml:snakeyaml:1.26'
diff --git a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java
index 4c2f24626c0..4c705ec451d 100644
--- a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java
+++ b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java
@@ -26,23 +26,6 @@ import static java.util.Arrays.asList;
 import static java.util.Collections.unmodifiableSet;
 
 public final class SqTables {
-  /**
-   * These tables are still involved in DB migrations, so potentially
-   * incorrect collation must be fixed so that joins with other
-   * tables are possible.
-   */
-  public static final Set<String> OLD_DROPPED_TABLES = unmodifiableSet(new HashSet<>(asList(
-    "active_dashboards",
-    "activities",
-    "dashboards",
-    "issue_filters",
-    "issue_filter_favourites",
-    "loaded_templates",
-    "measure_filters",
-    "measure_filter_favourites",
-    "resource_index",
-    "widgets",
-    "widget_properties")));
 
   /**
    * List of all the tables.
diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java
index a0695f79fc8..5666d5cf938 100644
--- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java
+++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java
@@ -81,7 +81,7 @@ public class ColumnDef {
 
   public boolean isInSonarQubeTable() {
     String tableName = table.toLowerCase(Locale.ENGLISH);
-    return SqTables.TABLES.contains(tableName) || SqTables.OLD_DROPPED_TABLES.contains(tableName);
+    return SqTables.TABLES.contains(tableName);
   }
 
   public enum ColumnDefRowConverter implements SqlExecutor.RowConverter<ColumnDef> {
diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/charset/ColumnDefTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/charset/ColumnDefTest.java
index c9ab3f9d4a0..f1c50ff4866 100644
--- a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/charset/ColumnDefTest.java
+++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/charset/ColumnDefTest.java
@@ -43,9 +43,4 @@ public class ColumnDefTest {
     assertThat(underTest.isInSonarQubeTable()).isTrue();
   }
 
-  @Test
-  public void isInSonarQubeTable_returns_true_if_table_existed_in_previous_versions_of_sonarqube() {
-    ColumnDef underTest = new ColumnDef("activities", "kee", "charset", "collation", "NVARCHAR", 100L, false);
-    assertThat(underTest.isInSonarQubeTable()).isTrue();
-  }
 }
diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/BackendCleanup.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/BackendCleanup.java
deleted file mode 100644
index b7bbd9b0782..00000000000
--- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/BackendCleanup.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2020 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.platform;
-
-import com.google.common.collect.ImmutableMap;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import org.sonar.api.server.ServerSide;
-import org.sonar.api.utils.log.Loggers;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.dialect.Oracle;
-import org.sonar.db.version.SqTables;
-import org.sonar.server.component.index.ComponentIndexDefinition;
-import org.sonar.server.es.BulkIndexer;
-import org.sonar.server.es.EsClient;
-import org.sonar.server.es.Index;
-import org.sonar.server.es.IndexType;
-import org.sonar.server.issue.index.IssueIndexDefinition;
-import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition;
-import org.sonar.server.property.InternalProperties;
-import org.sonar.server.view.index.ViewIndexDefinition;
-
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-
-@ServerSide
-public class BackendCleanup {
-
-  private static final String[] ANALYSIS_TABLES = {
-    "ce_activity", "ce_queue", "ce_task_input", "ce_scanner_context",
-    "duplications_index", "events", "issues", "issue_changes", "manual_measures",
-    "notifications", "project_links", "project_measures", "components", "projects",
-    "snapshots", "file_sources", "webhook_deliveries"
-  };
-  private static final String[] COMPONENT_RELATED_TABLES = {
-    "group_roles", "user_roles", "properties"
-  };
-  private static final Map<String, TableCleaner> TABLE_CLEANERS = ImmutableMap.of(
-    "organizations", BackendCleanup::truncateOrganizations,
-    "users", BackendCleanup::truncateUsers,
-    "groups", BackendCleanup::truncateGroups,
-    "internal_properties", BackendCleanup::truncateInternalProperties,
-    "schema_migrations", BackendCleanup::truncateSchemaMigrations);
-
-  private final EsClient esClient;
-  private final DbClient dbClient;
-
-  public BackendCleanup(EsClient esClient, DbClient dbClient) {
-    this.esClient = esClient;
-    this.dbClient = dbClient;
-  }
-
-  public void clearAll() {
-    clearDb();
-    clearIndexes();
-  }
-
-  public void clearDb() {
-    try (DbSession dbSession = dbClient.openSession(false);
-      Connection connection = dbSession.getConnection();
-      Statement ddlStatement = connection.createStatement()) {
-      for (String tableName : SqTables.TABLES) {
-        Optional.ofNullable(TABLE_CLEANERS.get(tableName))
-          .orElse(BackendCleanup::truncateDefault)
-          .clean(tableName, ddlStatement, connection);
-      }
-    } catch (Exception e) {
-      throw new IllegalStateException("Fail to clear db", e);
-    }
-  }
-
-  public void clearIndexes() {
-    Loggers.get(getClass()).info("Truncate Elasticsearch indices");
-    try {
-      esClient.prepareClearCache().get();
-
-      for (String index : esClient.prepareState().get().getState().getMetaData().getConcreteAllIndices()) {
-        /*under the hood, type is not used to perform clearIndex, so it's ok it does not match any existing index*/
-        clearIndex(Index.simple(index));
-      }
-    } catch (Exception e) {
-      throw new IllegalStateException("Unable to clear indexes", e);
-    }
-  }
-
-  /**
-   * Reset data in order to to be in same state as a fresh installation (but without having to drop db and restart the server).
-   *
-   * Please be careful when updating this method as it's called by Orchestrator.
-   */
-  public void resetData() {
-    try (DbSession dbSession = dbClient.openSession(false);
-      Connection connection = dbSession.getConnection()) {
-
-      truncateAnalysisTables(connection);
-      deleteManualRules(connection);
-      truncateInternalProperties(null, null, connection);
-      truncateUsers(null, null, connection);
-      truncateOrganizations(null, null, connection);
-    } catch (SQLException e) {
-      throw new IllegalStateException("Fail to reset data", e);
-    }
-
-    clearIndex(IssueIndexDefinition.DESCRIPTOR);
-    clearIndex(ViewIndexDefinition.DESCRIPTOR);
-    clearIndex(ProjectMeasuresIndexDefinition.DESCRIPTOR);
-    clearIndex(ComponentIndexDefinition.DESCRIPTOR);
-  }
-
-  private void truncateAnalysisTables(Connection connection) throws SQLException {
-    try (Statement statement = connection.createStatement()) {
-      // Clear inspection tables
-      for (String table : ANALYSIS_TABLES) {
-        statement.execute(createTruncateSql(table.toLowerCase(Locale.ENGLISH)));
-        // commit is useless on some databases
-        connection.commit();
-      }
-      for (String table : COMPONENT_RELATED_TABLES) {
-        statement.execute("DELETE FROM " + table + " WHERE component_uuid IS NOT NULL");
-        connection.commit();
-      }
-    }
-  }
-
-  private String createTruncateSql(String table) {
-    if (dbClient.getDatabase().getDialect().getId().equals(Oracle.ID)) {
-      // truncate operation is needs to lock the table on Oracle. Unfortunately
-      // it fails sometimes in our QA environment because table is locked.
-      // We never found the root cause (no locks found when displaying them just after
-      // receiving the error).
-      // Workaround is to use "delete" operation. It does not require lock on table.
-      return "DELETE FROM " + table;
-    }
-    return "TRUNCATE TABLE " + table;
-  }
-
-  private static void deleteManualRules(Connection connection) throws SQLException {
-    try (PreparedStatement statement = connection.prepareStatement("DELETE FROM rules WHERE rules.plugin_name='manual'")) {
-      statement.execute();
-      // commit is useless on some databases
-      connection.commit();
-    }
-  }
-
-  /**
-   * Completely remove a index with all types
-   */
-  public void clearIndex(Index index) {
-    BulkIndexer.delete(esClient, IndexType.main(index, index.getName()), esClient.prepareSearch(index).setQuery(matchAllQuery()));
-  }
-
-  @FunctionalInterface
-  private interface TableCleaner {
-    void clean(String tableName, Statement ddlStatement, Connection connection) throws SQLException;
-  }
-
-  private static void truncateDefault(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
-    ddlStatement.execute("TRUNCATE TABLE " + tableName.toLowerCase(Locale.ENGLISH));
-    // commit is useless on some databases
-    connection.commit();
-  }
-
-  /**
-   * Default organization must never be deleted
-   */
-  private static void truncateOrganizations(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
-    try (PreparedStatement preparedStatement = connection.prepareStatement("delete from organizations where kee <> ?")) {
-      preparedStatement.setString(1, "default-organization");
-      preparedStatement.execute();
-      // commit is useless on some databases
-      connection.commit();
-    }
-  }
-
-  /**
-   * User admin must never be deleted.
-   */
-  private static void truncateUsers(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
-    try (PreparedStatement preparedStatement = connection.prepareStatement("delete from users where login <> ?")) {
-      preparedStatement.setString(1, "admin");
-      preparedStatement.execute();
-      // commit is useless on some databases
-      connection.commit();
-    }
-    // "admin" is not flagged as root by default
-    try (PreparedStatement preparedStatement = connection.prepareStatement("update users set is_root=?")) {
-      preparedStatement.setBoolean(1, false);
-      preparedStatement.execute();
-      // commit is useless on some databases
-      connection.commit();
-    }
-  }
-
-  /**
-   * Groups sonar-users is referenced by the default organization as its default group.
-   */
-  private static void truncateGroups(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
-    try (PreparedStatement preparedStatement = connection.prepareStatement("delete from groups where name <> ?")) {
-      preparedStatement.setString(1, "sonar-users");
-      preparedStatement.execute();
-      // commit is useless on some databases
-      connection.commit();
-    }
-  }
-
-  /**
-   * Internal property {@link InternalProperties#DEFAULT_ORGANIZATION} must never be deleted.
-   */
-  private static void truncateInternalProperties(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
-    try (PreparedStatement preparedStatement = connection.prepareStatement("delete from internal_properties where kee not in (?,?,?,?,?)")) {
-      preparedStatement.setString(1, InternalProperties.DEFAULT_ORGANIZATION);
-      preparedStatement.setString(2, InternalProperties.SERVER_ID_CHECKSUM);
-      preparedStatement.setString(3, InternalProperties.DEFAULT_PROJECT_TEMPLATE);
-      preparedStatement.setString(4, InternalProperties.DEFAULT_PORTFOLIO_TEMPLATE);
-      preparedStatement.setString(5, InternalProperties.DEFAULT_APPLICATION_TEMPLATE);
-      preparedStatement.execute();
-      // commit is useless on some databases
-      connection.commit();
-    }
-  }
-
-  /**
-   * Data in SCHEMA_MIGRATIONS table is inserted when DB is created and should not be altered afterwards.
-   */
-  private static void truncateSchemaMigrations(String tableName, Statement ddlStatement, Connection connection) {
-    // do nothing
-  }
-
-}
diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/BackendCleanupTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/BackendCleanupTest.java
deleted file mode 100644
index dd8173b2367..00000000000
--- a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/BackendCleanupTest.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2020 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.platform;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.sonar.api.utils.System2;
-import org.sonar.core.util.Uuids;
-import org.sonar.db.DbTester;
-import org.sonar.db.component.ComponentDto;
-import org.sonar.db.component.ComponentTesting;
-import org.sonar.db.organization.OrganizationDto;
-import org.sonar.db.organization.OrganizationTesting;
-import org.sonar.db.property.PropertyDto;
-import org.sonar.db.rule.RuleTesting;
-import org.sonar.server.component.index.ComponentDoc;
-import org.sonar.server.component.index.ComponentIndexDefinition;
-import org.sonar.server.component.index.ComponentIndexer;
-import org.sonar.server.es.EsTester;
-import org.sonar.server.issue.IssueDocTesting;
-import org.sonar.server.issue.index.IssueIndexDefinition;
-import org.sonar.server.measure.index.ProjectMeasuresDoc;
-import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition;
-import org.sonar.server.rule.index.RuleDoc;
-import org.sonar.server.rule.index.RuleIndexDefinition;
-import org.sonar.server.view.index.ViewDoc;
-import org.sonar.server.view.index.ViewIndexDefinition;
-
-import static com.google.common.collect.Lists.newArrayList;
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class BackendCleanupTest {
-
-  @Rule
-  public EsTester es = EsTester.create();
-
-  @Rule
-  public DbTester dbTester = DbTester.create(System2.INSTANCE);
-
-  private BackendCleanup underTest = new BackendCleanup(es.client(), dbTester.getDbClient());
-  private OrganizationDto organization;
-
-  @Before
-  public void setUp() {
-    organization = OrganizationTesting.newOrganizationDto();
-  }
-
-  @Test
-  public void clear_db() {
-    insertSomeData();
-
-    underTest.clearDb();
-
-    assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0);
-    assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0);
-    assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0);
-    assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0);
-    assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0);
-  }
-
-  @Test
-  public void clear_indexes() {
-    es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
-    es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
-    es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
-
-    underTest.clearIndexes();
-
-    assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0);
-    assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0);
-  }
-
-  @Test
-  public void clear_all() {
-    insertSomeData();
-
-    es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
-    es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
-    es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
-
-    underTest.clearAll();
-
-    assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0);
-    assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(0);
-    assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0);
-
-    assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0);
-    assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0);
-    assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0);
-    assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0);
-    assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0);
-  }
-
-  @Test
-  public void reset_data() {
-    insertSomeData();
-
-    es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
-    es.putDocuments(ViewIndexDefinition.TYPE_VIEW, new ViewDoc().setUuid("CDEF").setProjects(newArrayList("DEFG")));
-    es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
-    es.putDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES, new ProjectMeasuresDoc()
-      .setId("PROJECT")
-      .setKey("Key")
-      .setName("Name"));
-    es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
-
-    underTest.resetData();
-
-    assertThat(dbTester.countRowsOfTable("projects")).isZero();
-    assertThat(dbTester.countRowsOfTable("components")).isZero();
-    assertThat(dbTester.countRowsOfTable("snapshots")).isZero();
-    assertThat(dbTester.countRowsOfTable("properties")).isZero();
-    assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isZero();
-    assertThat(es.countDocuments(ViewIndexDefinition.TYPE_VIEW)).isZero();
-    assertThat(es.countDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES)).isZero();
-    assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isZero();
-
-    // Rules should not be removed
-    assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(1);
-    assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(1);
-  }
-
-  private void insertSomeData() {
-    OrganizationDto organization = dbTester.organizations().insert();
-    ComponentDto project = dbTester.components().insertPrivateProject(organization);
-    dbTester.components().insertSnapshot(project);
-    dbTester.rules().insert();
-    dbTester.properties().insertProperty(new PropertyDto()
-      .setKey("sonar.profile.java")
-      .setValue("Sonar Way")
-      .setComponentUuid(project.uuid()));
-  }
-
-  private static RuleDoc newRuleDoc() {
-    return new RuleDoc().setUuid(Uuids.createFast()).setKey(RuleTesting.XOO_X1.toString()).setRepository(RuleTesting.XOO_X1.repository());
-  }
-
-  private ComponentDoc newComponentDoc() {
-    return ComponentIndexer.toDocument(ComponentTesting.newPrivateProjectDto(organization));
-  }
-}
diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java
index f1f0b4d26a3..23217a8527d 100644
--- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java
+++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java
@@ -119,7 +119,6 @@ import org.sonar.server.permission.PermissionUpdater;
 import org.sonar.server.permission.UserPermissionChanger;
 import org.sonar.server.permission.index.PermissionIndexer;
 import org.sonar.server.permission.ws.PermissionsWsModule;
-import org.sonar.server.platform.BackendCleanup;
 import org.sonar.server.platform.ClusterVerification;
 import org.sonar.server.platform.PersistentSettings;
 import org.sonar.server.platform.SystemInfoWriterModule;
@@ -256,7 +255,6 @@ public class PlatformLevel4 extends PlatformLevel {
       DefaultResourceTypes.get(),
       SettingsChangeNotifier.class,
       ServerWs.class,
-      BackendCleanup.class,
       IndexDefinitions.class,
       WebAnalyticsLoaderImpl.class,