* SONAR-13913 Remove usage of Orchestrator#resetData * Remove ProjectAnalysisRule * Upgrade to Orchestrator 3.31 * SONAR-13913 Remove BackendCleanup class * Remove code related to no more existing tablestags/8.6.0.39681
@@ -370,7 +370,7 @@ subprojects { | |||
dependency 'org.postgresql:postgresql:42.2.17' | |||
dependency 'org.reflections:reflections:0.9.12' | |||
dependency 'org.simpleframework:simple:4.1.21' | |||
dependency 'org.sonarsource.orchestrator:sonar-orchestrator:3.30.0.2630' | |||
dependency 'org.sonarsource.orchestrator:sonar-orchestrator:3.31.0.2646' | |||
dependency 'org.sonarsource.update-center:sonar-update-center-common:1.23.0.723' | |||
dependency 'org.subethamail:subethasmtp:3.1.7' | |||
dependency 'org.yaml:snakeyaml:1.26' |
@@ -26,23 +26,6 @@ import static java.util.Arrays.asList; | |||
import static java.util.Collections.unmodifiableSet; | |||
public final class SqTables { | |||
/** | |||
* These tables are still involved in DB migrations, so potentially | |||
* incorrect collation must be fixed so that joins with other | |||
* tables are possible. | |||
*/ | |||
public static final Set<String> OLD_DROPPED_TABLES = unmodifiableSet(new HashSet<>(asList( | |||
"active_dashboards", | |||
"activities", | |||
"dashboards", | |||
"issue_filters", | |||
"issue_filter_favourites", | |||
"loaded_templates", | |||
"measure_filters", | |||
"measure_filter_favourites", | |||
"resource_index", | |||
"widgets", | |||
"widget_properties"))); | |||
/** | |||
* List of all the tables. |
@@ -81,7 +81,7 @@ public class ColumnDef { | |||
public boolean isInSonarQubeTable() { | |||
String tableName = table.toLowerCase(Locale.ENGLISH); | |||
return SqTables.TABLES.contains(tableName) || SqTables.OLD_DROPPED_TABLES.contains(tableName); | |||
return SqTables.TABLES.contains(tableName); | |||
} | |||
public enum ColumnDefRowConverter implements SqlExecutor.RowConverter<ColumnDef> { |
@@ -43,9 +43,4 @@ public class ColumnDefTest { | |||
assertThat(underTest.isInSonarQubeTable()).isTrue(); | |||
} | |||
@Test | |||
public void isInSonarQubeTable_returns_true_if_table_existed_in_previous_versions_of_sonarqube() { | |||
ColumnDef underTest = new ColumnDef("activities", "kee", "charset", "collation", "NVARCHAR", 100L, false); | |||
assertThat(underTest.isInSonarQubeTable()).isTrue(); | |||
} | |||
} |
@@ -1,251 +0,0 @@ | |||
/* | |||
* SonarQube | |||
* Copyright (C) 2009-2020 SonarSource SA | |||
* mailto:info AT sonarsource DOT com | |||
* | |||
* This program is free software; you can redistribute it and/or | |||
* modify it under the terms of the GNU Lesser General Public | |||
* License as published by the Free Software Foundation; either | |||
* version 3 of the License, or (at your option) any later version. | |||
* | |||
* This program is distributed in the hope that it will be useful, | |||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||
* Lesser General Public License for more details. | |||
* | |||
* You should have received a copy of the GNU Lesser General Public License | |||
* along with this program; if not, write to the Free Software Foundation, | |||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
*/ | |||
package org.sonar.server.platform; | |||
import com.google.common.collect.ImmutableMap; | |||
import java.sql.Connection; | |||
import java.sql.PreparedStatement; | |||
import java.sql.SQLException; | |||
import java.sql.Statement; | |||
import java.util.Locale; | |||
import java.util.Map; | |||
import java.util.Optional; | |||
import org.sonar.api.server.ServerSide; | |||
import org.sonar.api.utils.log.Loggers; | |||
import org.sonar.db.DbClient; | |||
import org.sonar.db.DbSession; | |||
import org.sonar.db.dialect.Oracle; | |||
import org.sonar.db.version.SqTables; | |||
import org.sonar.server.component.index.ComponentIndexDefinition; | |||
import org.sonar.server.es.BulkIndexer; | |||
import org.sonar.server.es.EsClient; | |||
import org.sonar.server.es.Index; | |||
import org.sonar.server.es.IndexType; | |||
import org.sonar.server.issue.index.IssueIndexDefinition; | |||
import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition; | |||
import org.sonar.server.property.InternalProperties; | |||
import org.sonar.server.view.index.ViewIndexDefinition; | |||
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; | |||
@ServerSide | |||
public class BackendCleanup { | |||
private static final String[] ANALYSIS_TABLES = { | |||
"ce_activity", "ce_queue", "ce_task_input", "ce_scanner_context", | |||
"duplications_index", "events", "issues", "issue_changes", "manual_measures", | |||
"notifications", "project_links", "project_measures", "components", "projects", | |||
"snapshots", "file_sources", "webhook_deliveries" | |||
}; | |||
private static final String[] COMPONENT_RELATED_TABLES = { | |||
"group_roles", "user_roles", "properties" | |||
}; | |||
private static final Map<String, TableCleaner> TABLE_CLEANERS = ImmutableMap.of( | |||
"organizations", BackendCleanup::truncateOrganizations, | |||
"users", BackendCleanup::truncateUsers, | |||
"groups", BackendCleanup::truncateGroups, | |||
"internal_properties", BackendCleanup::truncateInternalProperties, | |||
"schema_migrations", BackendCleanup::truncateSchemaMigrations); | |||
private final EsClient esClient; | |||
private final DbClient dbClient; | |||
public BackendCleanup(EsClient esClient, DbClient dbClient) { | |||
this.esClient = esClient; | |||
this.dbClient = dbClient; | |||
} | |||
public void clearAll() { | |||
clearDb(); | |||
clearIndexes(); | |||
} | |||
public void clearDb() { | |||
try (DbSession dbSession = dbClient.openSession(false); | |||
Connection connection = dbSession.getConnection(); | |||
Statement ddlStatement = connection.createStatement()) { | |||
for (String tableName : SqTables.TABLES) { | |||
Optional.ofNullable(TABLE_CLEANERS.get(tableName)) | |||
.orElse(BackendCleanup::truncateDefault) | |||
.clean(tableName, ddlStatement, connection); | |||
} | |||
} catch (Exception e) { | |||
throw new IllegalStateException("Fail to clear db", e); | |||
} | |||
} | |||
public void clearIndexes() { | |||
Loggers.get(getClass()).info("Truncate Elasticsearch indices"); | |||
try { | |||
esClient.prepareClearCache().get(); | |||
for (String index : esClient.prepareState().get().getState().getMetaData().getConcreteAllIndices()) { | |||
/*under the hood, type is not used to perform clearIndex, so it's ok it does not match any existing index*/ | |||
clearIndex(Index.simple(index)); | |||
} | |||
} catch (Exception e) { | |||
throw new IllegalStateException("Unable to clear indexes", e); | |||
} | |||
} | |||
/** | |||
* Reset data in order to to be in same state as a fresh installation (but without having to drop db and restart the server). | |||
* | |||
* Please be careful when updating this method as it's called by Orchestrator. | |||
*/ | |||
public void resetData() { | |||
try (DbSession dbSession = dbClient.openSession(false); | |||
Connection connection = dbSession.getConnection()) { | |||
truncateAnalysisTables(connection); | |||
deleteManualRules(connection); | |||
truncateInternalProperties(null, null, connection); | |||
truncateUsers(null, null, connection); | |||
truncateOrganizations(null, null, connection); | |||
} catch (SQLException e) { | |||
throw new IllegalStateException("Fail to reset data", e); | |||
} | |||
clearIndex(IssueIndexDefinition.DESCRIPTOR); | |||
clearIndex(ViewIndexDefinition.DESCRIPTOR); | |||
clearIndex(ProjectMeasuresIndexDefinition.DESCRIPTOR); | |||
clearIndex(ComponentIndexDefinition.DESCRIPTOR); | |||
} | |||
private void truncateAnalysisTables(Connection connection) throws SQLException { | |||
try (Statement statement = connection.createStatement()) { | |||
// Clear inspection tables | |||
for (String table : ANALYSIS_TABLES) { | |||
statement.execute(createTruncateSql(table.toLowerCase(Locale.ENGLISH))); | |||
// commit is useless on some databases | |||
connection.commit(); | |||
} | |||
for (String table : COMPONENT_RELATED_TABLES) { | |||
statement.execute("DELETE FROM " + table + " WHERE component_uuid IS NOT NULL"); | |||
connection.commit(); | |||
} | |||
} | |||
} | |||
private String createTruncateSql(String table) { | |||
if (dbClient.getDatabase().getDialect().getId().equals(Oracle.ID)) { | |||
// truncate operation is needs to lock the table on Oracle. Unfortunately | |||
// it fails sometimes in our QA environment because table is locked. | |||
// We never found the root cause (no locks found when displaying them just after | |||
// receiving the error). | |||
// Workaround is to use "delete" operation. It does not require lock on table. | |||
return "DELETE FROM " + table; | |||
} | |||
return "TRUNCATE TABLE " + table; | |||
} | |||
private static void deleteManualRules(Connection connection) throws SQLException { | |||
try (PreparedStatement statement = connection.prepareStatement("DELETE FROM rules WHERE rules.plugin_name='manual'")) { | |||
statement.execute(); | |||
// commit is useless on some databases | |||
connection.commit(); | |||
} | |||
} | |||
/** | |||
* Completely remove a index with all types | |||
*/ | |||
public void clearIndex(Index index) { | |||
BulkIndexer.delete(esClient, IndexType.main(index, index.getName()), esClient.prepareSearch(index).setQuery(matchAllQuery())); | |||
} | |||
@FunctionalInterface | |||
private interface TableCleaner { | |||
void clean(String tableName, Statement ddlStatement, Connection connection) throws SQLException; | |||
} | |||
private static void truncateDefault(String tableName, Statement ddlStatement, Connection connection) throws SQLException { | |||
ddlStatement.execute("TRUNCATE TABLE " + tableName.toLowerCase(Locale.ENGLISH)); | |||
// commit is useless on some databases | |||
connection.commit(); | |||
} | |||
/** | |||
* Default organization must never be deleted | |||
*/ | |||
private static void truncateOrganizations(String tableName, Statement ddlStatement, Connection connection) throws SQLException { | |||
try (PreparedStatement preparedStatement = connection.prepareStatement("delete from organizations where kee <> ?")) { | |||
preparedStatement.setString(1, "default-organization"); | |||
preparedStatement.execute(); | |||
// commit is useless on some databases | |||
connection.commit(); | |||
} | |||
} | |||
/** | |||
* User admin must never be deleted. | |||
*/ | |||
private static void truncateUsers(String tableName, Statement ddlStatement, Connection connection) throws SQLException { | |||
try (PreparedStatement preparedStatement = connection.prepareStatement("delete from users where login <> ?")) { | |||
preparedStatement.setString(1, "admin"); | |||
preparedStatement.execute(); | |||
// commit is useless on some databases | |||
connection.commit(); | |||
} | |||
// "admin" is not flagged as root by default | |||
try (PreparedStatement preparedStatement = connection.prepareStatement("update users set is_root=?")) { | |||
preparedStatement.setBoolean(1, false); | |||
preparedStatement.execute(); | |||
// commit is useless on some databases | |||
connection.commit(); | |||
} | |||
} | |||
/** | |||
* Groups sonar-users is referenced by the default organization as its default group. | |||
*/ | |||
private static void truncateGroups(String tableName, Statement ddlStatement, Connection connection) throws SQLException { | |||
try (PreparedStatement preparedStatement = connection.prepareStatement("delete from groups where name <> ?")) { | |||
preparedStatement.setString(1, "sonar-users"); | |||
preparedStatement.execute(); | |||
// commit is useless on some databases | |||
connection.commit(); | |||
} | |||
} | |||
/** | |||
* Internal property {@link InternalProperties#DEFAULT_ORGANIZATION} must never be deleted. | |||
*/ | |||
private static void truncateInternalProperties(String tableName, Statement ddlStatement, Connection connection) throws SQLException { | |||
try (PreparedStatement preparedStatement = connection.prepareStatement("delete from internal_properties where kee not in (?,?,?,?,?)")) { | |||
preparedStatement.setString(1, InternalProperties.DEFAULT_ORGANIZATION); | |||
preparedStatement.setString(2, InternalProperties.SERVER_ID_CHECKSUM); | |||
preparedStatement.setString(3, InternalProperties.DEFAULT_PROJECT_TEMPLATE); | |||
preparedStatement.setString(4, InternalProperties.DEFAULT_PORTFOLIO_TEMPLATE); | |||
preparedStatement.setString(5, InternalProperties.DEFAULT_APPLICATION_TEMPLATE); | |||
preparedStatement.execute(); | |||
// commit is useless on some databases | |||
connection.commit(); | |||
} | |||
} | |||
/** | |||
* Data in SCHEMA_MIGRATIONS table is inserted when DB is created and should not be altered afterwards. | |||
*/ | |||
private static void truncateSchemaMigrations(String tableName, Statement ddlStatement, Connection connection) { | |||
// do nothing | |||
} | |||
} |
@@ -1,159 +0,0 @@ | |||
/* | |||
* SonarQube | |||
* Copyright (C) 2009-2020 SonarSource SA | |||
* mailto:info AT sonarsource DOT com | |||
* | |||
* This program is free software; you can redistribute it and/or | |||
* modify it under the terms of the GNU Lesser General Public | |||
* License as published by the Free Software Foundation; either | |||
* version 3 of the License, or (at your option) any later version. | |||
* | |||
* This program is distributed in the hope that it will be useful, | |||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||
* Lesser General Public License for more details. | |||
* | |||
* You should have received a copy of the GNU Lesser General Public License | |||
* along with this program; if not, write to the Free Software Foundation, | |||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
*/ | |||
package org.sonar.server.platform; | |||
import org.junit.Before; | |||
import org.junit.Rule; | |||
import org.junit.Test; | |||
import org.sonar.api.utils.System2; | |||
import org.sonar.core.util.Uuids; | |||
import org.sonar.db.DbTester; | |||
import org.sonar.db.component.ComponentDto; | |||
import org.sonar.db.component.ComponentTesting; | |||
import org.sonar.db.organization.OrganizationDto; | |||
import org.sonar.db.organization.OrganizationTesting; | |||
import org.sonar.db.property.PropertyDto; | |||
import org.sonar.db.rule.RuleTesting; | |||
import org.sonar.server.component.index.ComponentDoc; | |||
import org.sonar.server.component.index.ComponentIndexDefinition; | |||
import org.sonar.server.component.index.ComponentIndexer; | |||
import org.sonar.server.es.EsTester; | |||
import org.sonar.server.issue.IssueDocTesting; | |||
import org.sonar.server.issue.index.IssueIndexDefinition; | |||
import org.sonar.server.measure.index.ProjectMeasuresDoc; | |||
import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition; | |||
import org.sonar.server.rule.index.RuleDoc; | |||
import org.sonar.server.rule.index.RuleIndexDefinition; | |||
import org.sonar.server.view.index.ViewDoc; | |||
import org.sonar.server.view.index.ViewIndexDefinition; | |||
import static com.google.common.collect.Lists.newArrayList; | |||
import static org.assertj.core.api.Assertions.assertThat; | |||
public class BackendCleanupTest { | |||
@Rule | |||
public EsTester es = EsTester.create(); | |||
@Rule | |||
public DbTester dbTester = DbTester.create(System2.INSTANCE); | |||
private BackendCleanup underTest = new BackendCleanup(es.client(), dbTester.getDbClient()); | |||
private OrganizationDto organization; | |||
@Before | |||
public void setUp() { | |||
organization = OrganizationTesting.newOrganizationDto(); | |||
} | |||
@Test | |||
public void clear_db() { | |||
insertSomeData(); | |||
underTest.clearDb(); | |||
assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0); | |||
assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0); | |||
assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0); | |||
assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0); | |||
assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0); | |||
} | |||
@Test | |||
public void clear_indexes() { | |||
es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc()); | |||
es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc()); | |||
es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc()); | |||
underTest.clearIndexes(); | |||
assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0); | |||
assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0); | |||
} | |||
@Test | |||
public void clear_all() { | |||
insertSomeData(); | |||
es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc()); | |||
es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc()); | |||
es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc()); | |||
underTest.clearAll(); | |||
assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0); | |||
assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(0); | |||
assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0); | |||
assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0); | |||
assertThat(dbTester.countRowsOfTable("components")).isEqualTo(0); | |||
assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0); | |||
assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(0); | |||
assertThat(dbTester.countRowsOfTable("properties")).isEqualTo(0); | |||
} | |||
@Test | |||
public void reset_data() { | |||
insertSomeData(); | |||
es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc()); | |||
es.putDocuments(ViewIndexDefinition.TYPE_VIEW, new ViewDoc().setUuid("CDEF").setProjects(newArrayList("DEFG"))); | |||
es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc()); | |||
es.putDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES, new ProjectMeasuresDoc() | |||
.setId("PROJECT") | |||
.setKey("Key") | |||
.setName("Name")); | |||
es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc()); | |||
underTest.resetData(); | |||
assertThat(dbTester.countRowsOfTable("projects")).isZero(); | |||
assertThat(dbTester.countRowsOfTable("components")).isZero(); | |||
assertThat(dbTester.countRowsOfTable("snapshots")).isZero(); | |||
assertThat(dbTester.countRowsOfTable("properties")).isZero(); | |||
assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isZero(); | |||
assertThat(es.countDocuments(ViewIndexDefinition.TYPE_VIEW)).isZero(); | |||
assertThat(es.countDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES)).isZero(); | |||
assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isZero(); | |||
// Rules should not be removed | |||
assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(1); | |||
assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(1); | |||
} | |||
private void insertSomeData() { | |||
OrganizationDto organization = dbTester.organizations().insert(); | |||
ComponentDto project = dbTester.components().insertPrivateProject(organization); | |||
dbTester.components().insertSnapshot(project); | |||
dbTester.rules().insert(); | |||
dbTester.properties().insertProperty(new PropertyDto() | |||
.setKey("sonar.profile.java") | |||
.setValue("Sonar Way") | |||
.setComponentUuid(project.uuid())); | |||
} | |||
private static RuleDoc newRuleDoc() { | |||
return new RuleDoc().setUuid(Uuids.createFast()).setKey(RuleTesting.XOO_X1.toString()).setRepository(RuleTesting.XOO_X1.repository()); | |||
} | |||
private ComponentDoc newComponentDoc() { | |||
return ComponentIndexer.toDocument(ComponentTesting.newPrivateProjectDto(organization)); | |||
} | |||
} |
@@ -119,7 +119,6 @@ import org.sonar.server.permission.PermissionUpdater; | |||
import org.sonar.server.permission.UserPermissionChanger; | |||
import org.sonar.server.permission.index.PermissionIndexer; | |||
import org.sonar.server.permission.ws.PermissionsWsModule; | |||
import org.sonar.server.platform.BackendCleanup; | |||
import org.sonar.server.platform.ClusterVerification; | |||
import org.sonar.server.platform.PersistentSettings; | |||
import org.sonar.server.platform.SystemInfoWriterModule; | |||
@@ -256,7 +255,6 @@ public class PlatformLevel4 extends PlatformLevel { | |||
DefaultResourceTypes.get(), | |||
SettingsChangeNotifier.class, | |||
ServerWs.class, | |||
BackendCleanup.class, | |||
IndexDefinitions.class, | |||
WebAnalyticsLoaderImpl.class, | |||