--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.platform;
+
+import com.google.common.collect.ImmutableMap;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import org.sonar.api.server.ServerSide;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.dialect.Oracle;
+import org.sonar.db.version.SqTables;
+import org.sonar.server.component.index.ComponentIndexDefinition;
+import org.sonar.server.es.BulkIndexer;
+import org.sonar.server.es.EsClient;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.issue.index.IssueIndexDefinition;
+import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition;
+import org.sonar.server.property.InternalProperties;
+import org.sonar.server.view.index.ViewIndexDefinition;
+
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+
+@ServerSide
+public class BackendCleanup {
+
+ private static final String[] ANALYSIS_TABLES = {
+ "ce_activity", "ce_queue", "ce_task_input", "ce_scanner_context",
+ "duplications_index", "events", "issues", "issue_changes", "manual_measures",
+ "notifications", "project_links", "project_measures", "components", "projects",
+ "snapshots", "file_sources", "webhook_deliveries"
+ };
+ private static final String[] COMPONENT_RELATED_TABLES = {
+ "group_roles", "user_roles", "properties"
+ };
+ private static final Map<String, TableCleaner> TABLE_CLEANERS = ImmutableMap.of(
+ "organizations", BackendCleanup::truncateOrganizations,
+ "users", BackendCleanup::truncateUsers,
+ "groups", BackendCleanup::truncateGroups,
+ "internal_properties", BackendCleanup::truncateInternalProperties,
+ "schema_migrations", BackendCleanup::truncateSchemaMigrations);
+
+ private final EsClient esClient;
+ private final DbClient dbClient;
+
+ public BackendCleanup(EsClient esClient, DbClient dbClient) {
+ this.esClient = esClient;
+ this.dbClient = dbClient;
+ }
+
+ public void clearAll() {
+ clearDb();
+ clearIndexes();
+ }
+
+ public void clearDb() {
+ try (DbSession dbSession = dbClient.openSession(false);
+ Connection connection = dbSession.getConnection();
+ Statement ddlStatement = connection.createStatement()) {
+ for (String tableName : SqTables.TABLES) {
+ Optional.ofNullable(TABLE_CLEANERS.get(tableName))
+ .orElse(BackendCleanup::truncateDefault)
+ .clean(tableName, ddlStatement, connection);
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException("Fail to clear db", e);
+ }
+ }
+
+ public void clearIndexes() {
+ Loggers.get(getClass()).info("Truncate Elasticsearch indices");
+ try {
+ esClient.prepareClearCache().get();
+
+ for (String index : esClient.prepareState().get().getState().getMetaData().getConcreteAllIndices()) {
+ /*under the hood, type is not used to perform clearIndex, so it's ok it does not match any existing index*/
+ clearIndex(Index.simple(index));
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException("Unable to clear indexes", e);
+ }
+ }
+
+ /**
+ * Reset data in order to to be in same state as a fresh installation (but without having to drop db and restart the server).
+ *
+ * Please be careful when updating this method as it's called by Orchestrator.
+ */
+ public void resetData() {
+ try (DbSession dbSession = dbClient.openSession(false);
+ Connection connection = dbSession.getConnection()) {
+
+ truncateAnalysisTables(connection);
+ deleteManualRules(connection);
+ truncateInternalProperties(null, null, connection);
+ truncateUsers(null, null, connection);
+ truncateOrganizations(null, null, connection);
+ } catch (SQLException e) {
+ throw new IllegalStateException("Fail to reset data", e);
+ }
+
+ clearIndex(IssueIndexDefinition.DESCRIPTOR);
+ clearIndex(ViewIndexDefinition.DESCRIPTOR);
+ clearIndex(ProjectMeasuresIndexDefinition.DESCRIPTOR);
+ clearIndex(ComponentIndexDefinition.DESCRIPTOR);
+ }
+
+ private void truncateAnalysisTables(Connection connection) throws SQLException {
+ try (Statement statement = connection.createStatement()) {
+ // Clear inspection tables
+ for (String table : ANALYSIS_TABLES) {
+ statement.execute(createTruncateSql(table.toLowerCase(Locale.ENGLISH)));
+ // commit is useless on some databases
+ connection.commit();
+ }
+ for (String table : COMPONENT_RELATED_TABLES) {
+ statement.execute("DELETE FROM " + table + " WHERE component_uuid IS NOT NULL");
+ connection.commit();
+ }
+ }
+ }
+
+ private String createTruncateSql(String table) {
+ if (dbClient.getDatabase().getDialect().getId().equals(Oracle.ID)) {
+ // truncate operation is needs to lock the table on Oracle. Unfortunately
+ // it fails sometimes in our QA environment because table is locked.
+ // We never found the root cause (no locks found when displaying them just after
+ // receiving the error).
+ // Workaround is to use "delete" operation. It does not require lock on table.
+ return "DELETE FROM " + table;
+ }
+ return "TRUNCATE TABLE " + table;
+ }
+
+ private static void deleteManualRules(Connection connection) throws SQLException {
+ try (PreparedStatement statement = connection.prepareStatement("DELETE FROM rules WHERE rules.plugin_name='manual'")) {
+ statement.execute();
+ // commit is useless on some databases
+ connection.commit();
+ }
+ }
+
+ /**
+ * Completely remove a index with all types
+ */
+ public void clearIndex(Index index) {
+ BulkIndexer.delete(esClient, IndexType.main(index, index.getName()), esClient.prepareSearch(index).setQuery(matchAllQuery()));
+ }
+
+ @FunctionalInterface
+ private interface TableCleaner {
+ void clean(String tableName, Statement ddlStatement, Connection connection) throws SQLException;
+ }
+
+ private static void truncateDefault(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+ ddlStatement.execute("TRUNCATE TABLE " + tableName.toLowerCase(Locale.ENGLISH));
+ // commit is useless on some databases
+ connection.commit();
+ }
+
+ /**
+ * Default organization must never be deleted
+ */
+ private static void truncateOrganizations(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+ try (PreparedStatement preparedStatement = connection.prepareStatement("delete from organizations where kee <> ?")) {
+ preparedStatement.setString(1, "default-organization");
+ preparedStatement.execute();
+ // commit is useless on some databases
+ connection.commit();
+ }
+ }
+
+ /**
+ * User admin must never be deleted.
+ */
+ private static void truncateUsers(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+ try (PreparedStatement preparedStatement = connection.prepareStatement("delete from users where login <> ?")) {
+ preparedStatement.setString(1, "admin");
+ preparedStatement.execute();
+ // commit is useless on some databases
+ connection.commit();
+ }
+ // "admin" is not flagged as root by default
+ try (PreparedStatement preparedStatement = connection.prepareStatement("update users set is_root=?")) {
+ preparedStatement.setBoolean(1, false);
+ preparedStatement.execute();
+ // commit is useless on some databases
+ connection.commit();
+ }
+ }
+
+ /**
+ * Groups sonar-users is referenced by the default organization as its default group.
+ */
+ private static void truncateGroups(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+ try (PreparedStatement preparedStatement = connection.prepareStatement("delete from groups where name <> ?")) {
+ preparedStatement.setString(1, "sonar-users");
+ preparedStatement.execute();
+ // commit is useless on some databases
+ connection.commit();
+ }
+ }
+
+ /**
+ * Internal property {@link InternalProperties#DEFAULT_ORGANIZATION} must never be deleted.
+ */
+ private static void truncateInternalProperties(String tableName, Statement ddlStatement, Connection connection) throws SQLException {
+ try (PreparedStatement preparedStatement = connection.prepareStatement("delete from internal_properties where kee not in (?,?,?,?,?)")) {
+ preparedStatement.setString(1, InternalProperties.DEFAULT_ORGANIZATION);
+ preparedStatement.setString(2, InternalProperties.SERVER_ID_CHECKSUM);
+ preparedStatement.setString(3, InternalProperties.DEFAULT_PROJECT_TEMPLATE);
+ preparedStatement.setString(4, InternalProperties.DEFAULT_PORTFOLIO_TEMPLATE);
+ preparedStatement.setString(5, InternalProperties.DEFAULT_APPLICATION_TEMPLATE);
+ preparedStatement.execute();
+ // commit is useless on some databases
+ connection.commit();
+ }
+ }
+
+ /**
+ * Data in SCHEMA_MIGRATIONS table is inserted when DB is created and should not be altered afterwards.
+ */
+ private static void truncateSchemaMigrations(String tableName, Statement ddlStatement, Connection connection) {
+ // do nothing
+ }
+
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.platform;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.utils.System2;
+import org.sonar.core.util.Uuids;
+import org.sonar.db.DbTester;
+import org.sonar.db.component.ComponentDto;
+import org.sonar.db.component.ComponentTesting;
+import org.sonar.db.organization.OrganizationDto;
+import org.sonar.db.organization.OrganizationTesting;
+import org.sonar.db.property.PropertyDto;
+import org.sonar.db.rule.RuleTesting;
+import org.sonar.server.component.index.ComponentDoc;
+import org.sonar.server.component.index.ComponentIndexDefinition;
+import org.sonar.server.component.index.ComponentIndexer;
+import org.sonar.server.es.EsTester;
+import org.sonar.server.issue.IssueDocTesting;
+import org.sonar.server.issue.index.IssueIndexDefinition;
+import org.sonar.server.measure.index.ProjectMeasuresDoc;
+import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition;
+import org.sonar.server.rule.index.RuleDoc;
+import org.sonar.server.rule.index.RuleIndexDefinition;
+import org.sonar.server.view.index.ViewDoc;
+import org.sonar.server.view.index.ViewIndexDefinition;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class BackendCleanupTest {
+
+ @Rule
+ public EsTester es = EsTester.create();
+
+ @Rule
+ public DbTester dbTester = DbTester.create(System2.INSTANCE);
+
+ private BackendCleanup underTest = new BackendCleanup(es.client(), dbTester.getDbClient());
+ private OrganizationDto organization;
+
+ @Before
+ public void setUp() {
+ organization = OrganizationTesting.newOrganizationDto();
+ }
+
+ @Test
+ public void clear_db() {
+ insertSomeData();
+
+ underTest.clearDb();
+
+ assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0);
+ assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0);
+ assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0);
+ assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0);
+ assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0);
+ }
+
+ @Test
+ public void clear_indexes() {
+ es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
+ es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
+ es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
+
+ underTest.clearIndexes();
+
+ assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0);
+ assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0);
+ }
+
+ @Test
+ public void clear_all() {
+ insertSomeData();
+
+ es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
+ es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
+ es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
+
+ underTest.clearAll();
+
+ assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0);
+ assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(0);
+ assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0);
+
+ assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0);
+ assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0);
+ assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0);
+ assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0);
+ assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0);
+ }
+
+ @Test
+ public void reset_data() {
+ insertSomeData();
+
+ es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
+ es.putDocuments(ViewIndexDefinition.TYPE_VIEW, new ViewDoc().setUuid("CDEF").setProjects(newArrayList("DEFG")));
+ es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
+ es.putDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES, new ProjectMeasuresDoc()
+ .setId("PROJECT")
+ .setKey("Key")
+ .setName("Name"));
+ es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
+
+ underTest.resetData();
+
+ assertThat(dbTester.countRowsOfTable("projects")).isZero();
+ assertThat(dbTester.countRowsOfTable("components")).isZero();
+ assertThat(dbTester.countRowsOfTable("snapshots")).isZero();
+ assertThat(dbTester.countRowsOfTable("properties")).isZero();
+ assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isZero();
+ assertThat(es.countDocuments(ViewIndexDefinition.TYPE_VIEW)).isZero();
+ assertThat(es.countDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES)).isZero();
+ assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isZero();
+
+ // Rules should not be removed
+ assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(1);
+ assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(1);
+ }
+
+ private void insertSomeData() {
+ OrganizationDto organization = dbTester.organizations().insert();
+ ComponentDto project = dbTester.components().insertPrivateProject(organization);
+ dbTester.components().insertSnapshot(project);
+ dbTester.rules().insert();
+ dbTester.properties().insertProperty(new PropertyDto()
+ .setKey("sonar.profile.java")
+ .setValue("Sonar Way")
+ .setComponentUuid(project.uuid()));
+ }
+
+ private static RuleDoc newRuleDoc() {
+ return new RuleDoc().setUuid(Uuids.createFast()).setKey(RuleTesting.XOO_X1.toString()).setRepository(RuleTesting.XOO_X1.repository());
+ }
+
+ private ComponentDoc newComponentDoc() {
+ return ComponentIndexer.toDocument(ComponentTesting.newPrivateProjectDto(organization));
+ }
+}
projects / sonarqube.git / commitdiff