diff options
9 files changed, 270 insertions, 7 deletions
diff --git a/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasuresIT.java b/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasuresIT.java index 1bc0c0be4c7..e430671948e 100644 --- a/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasuresIT.java +++ b/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasuresIT.java @@ -23,8 +23,10 @@ import java.sql.SQLException; import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.Mockito; import org.sonar.core.util.UuidFactoryImpl; import org.sonar.db.MigrationDbTester; +import org.sonar.server.platform.db.migration.es.MigrationEsClient; import static org.assertj.core.api.Assertions.assertThat; @@ -33,7 +35,9 @@ class DeleteSoftwareQualityRatingFromProjectMeasuresIT { @RegisterExtension public final MigrationDbTester db = MigrationDbTester.createForMigrationStep(DeleteSoftwareQualityRatingFromProjectMeasures.class); - private final DeleteSoftwareQualityRatingFromProjectMeasures underTest = new DeleteSoftwareQualityRatingFromProjectMeasures(db.database()); + private final MigrationEsClient migrationEsClient = Mockito.mock(MigrationEsClient.class); + + private final DeleteSoftwareQualityRatingFromProjectMeasures underTest = new DeleteSoftwareQualityRatingFromProjectMeasures(db.database(), migrationEsClient); @Test void execute_whenMeasuresExists_shouldDeleteMeasures() throws SQLException { @@ -48,6 +52,7 @@ class DeleteSoftwareQualityRatingFromProjectMeasuresIT { underTest.execute(); assertThat(db.countSql("select count(1) from project_measures;")).isZero(); + Mockito.verify(migrationEsClient, Mockito.times(1)).deleteIndexes("projectmeasures"); } @Test diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java new file mode 100644 index 00000000000..cb01d7188a5 --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java @@ -0,0 +1,32 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.es; + +/** + * This class is used to delete es index in order to trigger reindexation + * when a migration step has changed the data used by the index. + */ +public interface MigrationEsClient { + + /** + * This method is re-entrant and does not fail if indexName or otherIndexNames do not exist + */ + void deleteIndexes(String name, String... otherNames); +} diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/package-info.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/package-info.java new file mode 100644 index 00000000000..9de338d20d3 --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.server.platform.db.migration.es; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasures.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasures.java index 92dfcaff2e8..8dd32588f1a 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasures.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasures.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.sonar.core.metric.SoftwareQualitiesMetrics; import org.sonar.db.Database; +import org.sonar.server.platform.db.migration.es.MigrationEsClient; import org.sonar.server.platform.db.migration.step.DataChange; import org.sonar.server.platform.db.migration.step.MassUpdate; @@ -44,7 +45,23 @@ public class DeleteSoftwareQualityRatingFromProjectMeasures extends DataChange { SoftwareQualitiesMetrics.SOFTWARE_QUALITY_RELIABILITY_REMEDIATION_EFFORT_KEY, SoftwareQualitiesMetrics.NEW_SOFTWARE_QUALITY_RELIABILITY_REMEDIATION_EFFORT_KEY, SoftwareQualitiesMetrics.SOFTWARE_QUALITY_MAINTAINABILITY_DEBT_RATIO_KEY, - SoftwareQualitiesMetrics.NEW_SOFTWARE_QUALITY_MAINTAINABILITY_DEBT_RATIO_KEY); + SoftwareQualitiesMetrics.NEW_SOFTWARE_QUALITY_MAINTAINABILITY_DEBT_RATIO_KEY, + + // Portfolios + "software_quality_reliability_rating_distribution", + "software_quality_security_rating_distribution", + "software_quality_maintainability_rating_distribution", + "new_software_quality_maintainability_rating_distribution", + "new_software_quality_reliability_rating_distribution", + "new_software_quality_security_rating_distribution", + + // Views + "last_change_on_software_quality_security_rating", + "last_change_on_software_quality_reliability_rating", + "last_change_on_software_quality_maintainability_rating", + "software_quality_security_rating_effort", + "software_quality_reliability_rating_effort", + "software_quality_maintainability_rating_effort"); private static final String SELECT_QUERY = """ select pm.uuid from project_measures pm @@ -52,8 +69,11 @@ public class DeleteSoftwareQualityRatingFromProjectMeasures extends DataChange { where m.name in (%s) """.formatted(SOFTWARE_QUALITY_METRICS_TO_DELETE.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","))); - public DeleteSoftwareQualityRatingFromProjectMeasures(Database db) { + private final MigrationEsClient migrationEsClient; + + public DeleteSoftwareQualityRatingFromProjectMeasures(Database db, MigrationEsClient migrationEsClient) { super(db); + this.migrationEsClient = migrationEsClient; } @Override @@ -66,5 +86,8 @@ public class DeleteSoftwareQualityRatingFromProjectMeasures extends DataChange { update.setString(1, row.getString(1)); return true; }); + + // Reindexation of project measures is required to align with removed values + migrationEsClient.deleteIndexes("projectmeasures"); } } diff --git a/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/NoOpMigrationEsClient.java b/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/NoOpMigrationEsClient.java new file mode 100644 index 00000000000..581ce5f0c0f --- /dev/null +++ b/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/NoOpMigrationEsClient.java @@ -0,0 +1,29 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db; + +import org.sonar.server.platform.db.migration.es.MigrationEsClient; + +public class NoOpMigrationEsClient implements MigrationEsClient { + @Override + public void deleteIndexes(String name, String... otherNames) { + + } +} diff --git a/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/SQDatabase.java b/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/SQDatabase.java index 29d3c8419d9..523deaee66e 100644 --- a/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/SQDatabase.java +++ b/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/SQDatabase.java @@ -186,6 +186,7 @@ public class SQDatabase extends DefaultDatabase { container.add(UuidFactoryFast.getInstance()); container.add(System2.INSTANCE); container.add(MapSettings.class); + container.add(NoOpMigrationEsClient.class); container.startComponents(); MigrationContainer migrationContainer = new MigrationContainerImpl(container, H2StepExecutor.class); diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java new file mode 100644 index 00000000000..eb8c4e2ae8f --- /dev/null +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java @@ -0,0 +1,57 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.server.platform.db.migration.es.MigrationEsClient; + +public class MigrationEsClientImpl implements MigrationEsClient { + private static final Logger LOG = LoggerFactory.getLogger(MigrationEsClientImpl.class); + private final EsClient client; + + public MigrationEsClientImpl(EsClient client) { + this.client = client; + } + + @Override + public void deleteIndexes(String name, String... otherNames) { + String[] indices = client.getIndex(new GetIndexRequest("_all")).getIndices(); + Set<String> existingIndices = Arrays.stream(indices).collect(Collectors.toSet()); + String[] toDelete = Stream.concat(Stream.of(name), Arrays.stream(otherNames)) + .distinct() + .filter(existingIndices::contains) + .toArray(String[]::new); + if (toDelete.length > 0) { + deleteIndex(toDelete); + } + } + + private void deleteIndex(String... indices) { + LOG.info("Drop Elasticsearch indices [{}]", String.join(",", indices)); + client.deleteIndex(new DeleteIndexRequest(indices)); + } +} diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java new file mode 100644 index 00000000000..4b42b6d6c47 --- /dev/null +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java @@ -0,0 +1,91 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import java.util.Iterator; +import org.elasticsearch.client.indices.GetMappingsRequest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.event.Level; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; +import org.sonar.server.platform.db.migration.es.MigrationEsClient; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; + +class MigrationEsClientImplTest { + @RegisterExtension + public LogTesterJUnit5 logTester = new LogTesterJUnit5(); + + @RegisterExtension + public EsTester es = EsTester.createCustom( + new SimpleIndexDefinition("as"), + new SimpleIndexDefinition("bs"), + new SimpleIndexDefinition("cs")); + + private final MigrationEsClient underTest = new MigrationEsClientImpl(es.client()); + + @Test + void delete_existing_index() { + underTest.deleteIndexes("as"); + + assertThat(loadExistingIndices()) + .toIterable() + .doesNotContain("as") + .contains("bs", "cs"); + assertThat(logTester.logs(Level.INFO)) + .contains("Drop Elasticsearch indices [as]"); + } + + @Test + void delete_index_that_does_not_exist() { + underTest.deleteIndexes("as", "xxx", "cs"); + + assertThat(loadExistingIndices()) + .toIterable() + .doesNotContain("as", "cs") + .contains("bs"); + assertThat(logTester.logs(Level.INFO)) + .contains("Drop Elasticsearch indices [as,cs]") + .doesNotContain("Drop Elasticsearch indices [xxx]"); + } + + private Iterator<String> loadExistingIndices() { + return es.client().getMapping(new GetMappingsRequest()).mappings().keySet().iterator(); + } + + private static class SimpleIndexDefinition implements IndexDefinition { + private final String indexName; + + public SimpleIndexDefinition(String indexName) { + this.indexName = indexName; + } + + @Override + public void define(IndexDefinitionContext context) { + IndexType.IndexMainType mainType = IndexType.main(Index.simple(indexName), indexName.substring(1)); + context.create( + mainType.getIndex(), + newBuilder(new MapSettings().asConfig()).build()) + .createTypeMapping(mainType); + } + } +} diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java index 3bb99f3731f..2e9be43668f 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java @@ -24,6 +24,7 @@ import org.sonar.core.extension.CoreExtensionsInstaller; import org.sonar.core.platform.PluginClassLoader; import org.sonar.core.platform.PluginClassloaderFactory; import org.sonar.core.platform.SpringComponentContainer; +import org.sonar.server.es.MigrationEsClientImpl; import org.sonar.server.l18n.ServerI18n; import org.sonar.server.platform.DatabaseServerCompatibility; import org.sonar.server.platform.DefaultServerUpgradeStatus; @@ -36,10 +37,6 @@ import org.sonar.server.platform.db.migration.DatabaseMigrationStateImpl; import org.sonar.server.platform.db.migration.MigrationConfigurationModule; import org.sonar.server.platform.db.migration.charset.DatabaseCharsetChecker; import org.sonar.server.platform.db.migration.version.DatabaseVersion; -import org.sonar.server.telemetry.TelemetryDbMigrationStepDurationProvider; -import org.sonar.server.telemetry.TelemetryDbMigrationSuccessProvider; -import org.sonar.server.telemetry.TelemetryDbMigrationStepsProvider; -import org.sonar.server.telemetry.TelemetryDbMigrationTotalTimeProvider; import org.sonar.server.platform.web.WebPagesCache; import org.sonar.server.plugins.InstalledPluginReferentialFactory; import org.sonar.server.plugins.PluginJarLoader; @@ -47,6 +44,10 @@ import org.sonar.server.plugins.ServerPluginJarExploder; import org.sonar.server.plugins.ServerPluginManager; import org.sonar.server.plugins.ServerPluginRepository; import org.sonar.server.plugins.WebServerExtensionInstaller; +import org.sonar.server.telemetry.TelemetryDbMigrationStepDurationProvider; +import org.sonar.server.telemetry.TelemetryDbMigrationStepsProvider; +import org.sonar.server.telemetry.TelemetryDbMigrationSuccessProvider; +import org.sonar.server.telemetry.TelemetryDbMigrationTotalTimeProvider; import static org.sonar.core.extension.CoreExtensionsInstaller.noAdditionalSideFilter; import static org.sonar.core.extension.PlatformLevelPredicates.hasPlatformLevel; @@ -60,6 +61,7 @@ public class PlatformLevel2 extends PlatformLevel { protected void configureLevel() { add( new MigrationConfigurationModule(), + MigrationEsClientImpl.class, DatabaseVersion.class, DatabaseServerCompatibility.class, |