From f135567f2bc832f2a24c4cf3d32d0b53d6f22cf8 Mon Sep 17 00:00:00 2001 From: Eric Giffon Date: Thu, 3 Oct 2024 11:22:53 +0200 Subject: [PATCH] SONAR-23213 Measures double write - purge --- .../container/ComputeEngineContainerImpl.java | 5 - .../sonar/ce/db/ReadOnlyPropertiesDao.java | 98 -------------- .../java/org/sonar/ce/db/package-info.java | 23 ---- .../ce/db/ReadOnlyPropertiesDaoTest.java | 124 ------------------ .../org/sonar/db/purge/PurgeCommands.java | 22 +++- .../java/org/sonar/db/purge/PurgeDao.java | 41 ++++-- .../java/org/sonar/db/purge/PurgeMapper.java | 6 + .../org/sonar/db/purge/PurgeMapper.xml | 20 ++- .../java/org/sonar/db/purge/PurgeDaoTest.java | 78 ++++++++++- .../sonar/db/purge/PurgeDaoWithAuditTest.java | 3 +- .../org/sonar/db/measure/MeasureDbTester.java | 12 ++ 11 files changed, 166 insertions(+), 266 deletions(-) delete mode 100644 server/sonar-ce/src/main/java/org/sonar/ce/db/ReadOnlyPropertiesDao.java delete mode 100644 server/sonar-ce/src/main/java/org/sonar/ce/db/package-info.java delete mode 100644 server/sonar-ce/src/test/java/org/sonar/ce/db/ReadOnlyPropertiesDaoTest.java diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java index e368e2ed137..1572c0a6257 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java @@ -48,7 +48,6 @@ import org.sonar.ce.StandaloneCeDistributedInformation; import org.sonar.ce.analysis.cache.cleaning.AnalysisCacheCleaningModule; import org.sonar.ce.async.SynchronousAsyncExecution; import org.sonar.ce.cleaning.CeCleaningModule; -import org.sonar.ce.db.ReadOnlyPropertiesDao; import org.sonar.ce.issue.index.NoAsyncIssueIndexing; import org.sonar.ce.logging.CeProcessLogging; import org.sonar.ce.monitoring.CEQueueStatusImpl; @@ -296,7 +295,6 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer { // DB new DaoModule(), - ReadOnlyPropertiesDao.class, DBSessionsImpl.class, DbClient.class, @@ -326,9 +324,6 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer { DatabaseSettingsEnabler.class, UrlSettings.class, - // add ReadOnlyPropertiesDao at level2 again so that it shadows PropertiesDao - ReadOnlyPropertiesDao.class, - // plugins PluginClassloaderFactory.class, CePluginJarExploder.class, diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/db/ReadOnlyPropertiesDao.java b/server/sonar-ce/src/main/java/org/sonar/ce/db/ReadOnlyPropertiesDao.java deleted file mode 100644 index 77df35e9cd9..00000000000 --- a/server/sonar-ce/src/main/java/org/sonar/ce/db/ReadOnlyPropertiesDao.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2024 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.ce.db; - -import java.util.Map; -import javax.annotation.Nullable; -import org.sonar.api.utils.System2; -import org.sonar.core.util.UuidFactory; -import org.sonar.db.DbSession; -import org.sonar.db.MyBatis; -import org.sonar.db.audit.NoOpAuditPersister; -import org.sonar.db.property.PropertiesDao; -import org.sonar.db.property.PropertyDto; - -/** - * Compute Engine specific override of {@link PropertiesDao} and {@link org.sonar.db.property.PropertiesDao} which - * implements no write method (ie. insert/update/delete) because updating the Properties is the Web Server responsibility - * alone. - *

- * This ugly trick is required because licensed plugin bundle {@link com.sonarsource.license.api.internal.ServerLicenseVerifierImpl} - * which update license properties by calling {@link PropertiesDao} directly and this can not be disabled. - *

- */ -public class ReadOnlyPropertiesDao extends PropertiesDao { - public ReadOnlyPropertiesDao(MyBatis mybatis, System2 system2, UuidFactory uuidFactory) { - super(mybatis, system2, uuidFactory, new NoOpAuditPersister()); - } - - @Override - public void saveProperty(DbSession session, PropertyDto property, @Nullable String userLogin, - @Nullable String projectKey, @Nullable String projectName, @Nullable String qualifier) { - // do nothing - } - - @Override - public void saveProperty(PropertyDto property) { - // do nothing - } - - @Override - public void deleteProjectProperty(String key, String projectUuid, String projectKey, String projectName, String qualifier) { - // do nothing - } - - @Override - public void deleteProjectProperty(DbSession session, String key, String projectUuid, String projectKey, - String projectName, String qualifier) { - // do nothing - } - - @Override - public void deleteProjectProperties(String key, String value, DbSession session) { - // do nothing - } - - @Override - public void deleteProjectProperties(String key, String value) { - // do nothing - } - - @Override - public void deleteGlobalProperty(String key, DbSession session) { - // do nothing - } - - @Override - public void deleteGlobalProperty(String key) { - // do nothing - } - - @Override - public void saveGlobalProperties(Map properties) { - // do nothing - } - - @Override - public void renamePropertyKey(String oldKey, String newKey) { - // do nothing - } - -} diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/db/package-info.java b/server/sonar-ce/src/main/java/org/sonar/ce/db/package-info.java deleted file mode 100644 index 4f8609c03e7..00000000000 --- a/server/sonar-ce/src/main/java/org/sonar/ce/db/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2024 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -@ParametersAreNonnullByDefault -package org.sonar.ce.db; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/db/ReadOnlyPropertiesDaoTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/db/ReadOnlyPropertiesDaoTest.java deleted file mode 100644 index 7e55ca74762..00000000000 --- a/server/sonar-ce/src/test/java/org/sonar/ce/db/ReadOnlyPropertiesDaoTest.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2024 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.ce.db; - -import org.junit.Test; -import org.sonar.api.utils.System2; -import org.sonar.core.util.SequenceUuidFactory; -import org.sonar.core.util.UuidFactory; -import org.sonar.db.DbSession; -import org.sonar.db.MyBatis; -import org.sonar.db.property.PropertyDto; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verifyNoMoreInteractions; - -public class ReadOnlyPropertiesDaoTest { - private MyBatis myBatis = mock(MyBatis.class); - private DbSession dbSession = mock(DbSession.class); - private PropertyDto propertyDto = mock(PropertyDto.class); - private PropertyDto oldPropertyDto = mock(PropertyDto.class); - private UuidFactory uuidFactory = new SequenceUuidFactory(); - private ReadOnlyPropertiesDao underTest = new ReadOnlyPropertiesDao(myBatis, System2.INSTANCE, uuidFactory); - - @Test - public void insertProperty() { - underTest.saveProperty(dbSession, propertyDto, null, null, null, null); - - assertNoInteraction(); - } - - @Test - public void insertProperty1() { - underTest.saveProperty(propertyDto); - - assertNoInteraction(); - } - - @Test - public void deleteProjectProperty() { - underTest.deleteProjectProperty(null, null, null, null, null, null); - - assertNoInteraction(); - - } - - @Test - public void deleteProjectProperty1() { - underTest.deleteProjectProperty(dbSession, null, null, null, null, null); - - assertNoInteraction(); - - } - - @Test - public void deleteProjectProperties() { - underTest.deleteProjectProperties(null, null); - - assertNoInteraction(); - - } - - @Test - public void deleteProjectProperties1() { - underTest.deleteProjectProperties(null, null, dbSession); - - assertNoInteraction(); - } - - @Test - public void deleteGlobalProperty() { - underTest.deleteGlobalProperty(null); - - assertNoInteraction(); - } - - @Test - public void deleteGlobalProperty1() { - underTest.deleteGlobalProperty(null, dbSession); - - assertNoInteraction(); - } - - @Test - public void insertGlobalProperties() { - underTest.saveGlobalProperties(null); - - assertNoInteraction(); - } - - @Test - public void renamePropertyKey() { - underTest.renamePropertyKey(null, null); - - assertNoInteraction(); - } - - @Test - public void saveProperty() { - underTest.saveProperty(oldPropertyDto); - - assertNoInteraction(); - } - - private void assertNoInteraction() { - verifyNoMoreInteractions(myBatis, dbSession, propertyDto); - } -} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java index eeb629b66cd..5b8996e43d9 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -138,7 +138,8 @@ class PurgeCommands { profiler.stop(); } - void purgeDisabledComponents(String rootComponentUuid, Collection disabledComponentUuids, PurgeListener listener) { + void purgeDisabledComponents(String rootComponentUuid, Collection disabledComponentUuids, PurgeListener listener, + boolean measuresMigrationEnabled) { Set missedDisabledComponentUuids = new HashSet<>(); profiler.start("purgeDisabledComponents (file_sources)"); @@ -171,6 +172,18 @@ class PurgeCommands { })); profiler.stop(); + if (measuresMigrationEnabled) { + profiler.start("purgeDisabledComponents (measures)"); + missedDisabledComponentUuids.addAll( + executeLargeInputs( + purgeMapper.selectDisabledComponentsWithJsonMeasures(rootComponentUuid), + input -> { + purgeMapper.deleteJsonMeasuresByComponentUuids(input); + return input; + })); + profiler.stop(); + } + session.commit(); // notify listener for any disabled component we found child data for which isn't part of the disabled components @@ -459,6 +472,13 @@ class PurgeCommands { profiler.stop(); } + void deleteJsonMeasures(String rootUuid) { + profiler.start("deleteJsonMeasures (measures)"); + purgeMapper.deleteJsonMeasuresByBranchUuid(rootUuid); + session.commit(); + profiler.stop(); + } + void deleteNewCodePeriods(String rootUuid) { profiler.start("deleteNewCodePeriods (new_code_periods)"); purgeMapper.deleteNewCodePeriodsByRootUuid(rootUuid); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java index 5220d33c257..7141bbdb326 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -40,10 +40,13 @@ import org.sonar.db.audit.model.ComponentNewValue; import org.sonar.db.component.BranchDto; import org.sonar.db.component.BranchMapper; import org.sonar.db.component.ComponentDto; +import org.sonar.db.property.PropertiesDao; +import org.sonar.db.property.PropertyDto; import static java.util.Collections.emptyList; import static java.util.Optional.ofNullable; import static org.sonar.api.utils.DateUtils.dateToLong; +import static org.sonar.core.config.CorePropertyDefinitions.SYSTEM_MEASURES_MIGRATION_ENABLED; import static org.sonar.db.DatabaseUtils.executeLargeInputs; public class PurgeDao implements Dao { @@ -54,10 +57,12 @@ public class PurgeDao implements Dao { private final System2 system2; private final AuditPersister auditPersister; + private final PropertiesDao propertiesDao; - public PurgeDao(System2 system2, AuditPersister auditPersister) { + public PurgeDao(System2 system2, AuditPersister auditPersister, PropertiesDao propertiesDao) { this.system2 = system2; this.auditPersister = auditPersister; + this.propertiesDao = propertiesDao; } public void purge(DbSession session, PurgeConfiguration conf, PurgeListener listener, PurgeProfiler profiler) { @@ -66,16 +71,18 @@ public class PurgeDao implements Dao { String rootUuid = conf.rootUuid(); deleteAbortedAnalyses(rootUuid, commands); purgeAnalyses(commands, rootUuid); - purgeDisabledComponents(commands, conf, listener); + boolean measuresMigrationEnabled = isMeasuresMigrationEnabled(); + purgeDisabledComponents(commands, conf, listener, measuresMigrationEnabled); deleteOldClosedIssues(conf, mapper, listener); purgeOldCeActivities(rootUuid, commands); purgeOldCeScannerContexts(rootUuid, commands); deleteOldDisabledComponents(commands, mapper, rootUuid); - purgeStaleBranches(commands, conf, mapper, rootUuid); + purgeStaleBranches(commands, conf, mapper, rootUuid, measuresMigrationEnabled); } - private static void purgeStaleBranches(PurgeCommands commands, PurgeConfiguration conf, PurgeMapper mapper, String rootUuid) { + private static void purgeStaleBranches(PurgeCommands commands, PurgeConfiguration conf, PurgeMapper mapper, String rootUuid, + boolean measuresMigrationEnabled) { Optional maxDate = conf.maxLiveDateOfInactiveBranches(); if (maxDate.isEmpty()) { // not available if branch plugin is not installed @@ -88,7 +95,7 @@ public class PurgeDao implements Dao { for (String branchUuid : branchUuids) { if (!rootUuid.equals(branchUuid)) { - deleteRootComponent(branchUuid, mapper, commands); + deleteRootComponent(branchUuid, mapper, commands, measuresMigrationEnabled); } } } @@ -101,10 +108,11 @@ public class PurgeDao implements Dao { commands.purgeAnalyses(analysisUuids); } - private static void purgeDisabledComponents(PurgeCommands commands, PurgeConfiguration conf, PurgeListener listener) { + private static void purgeDisabledComponents(PurgeCommands commands, PurgeConfiguration conf, PurgeListener listener, + boolean measuresMigrationEnabled) { String rootUuid = conf.rootUuid(); listener.onComponentsDisabling(rootUuid, conf.getDisabledComponentUuids()); - commands.purgeDisabledComponents(rootUuid, conf.getDisabledComponentUuids(), listener); + commands.purgeDisabledComponents(rootUuid, conf.getDisabledComponentUuids(), listener, measuresMigrationEnabled); } private static void deleteOldClosedIssues(PurgeConfiguration conf, PurgeMapper mapper, PurgeListener listener) { @@ -186,7 +194,7 @@ public class PurgeDao implements Dao { PurgeProfiler profiler = new PurgeProfiler(); PurgeMapper purgeMapper = mapper(session); PurgeCommands purgeCommands = new PurgeCommands(session, profiler, system2); - deleteRootComponent(uuid, purgeMapper, purgeCommands); + deleteRootComponent(uuid, purgeMapper, purgeCommands, isMeasuresMigrationEnabled()); } public void deleteProject(DbSession session, String uuid, String qualifier, String name, String key) { @@ -194,15 +202,16 @@ public class PurgeDao implements Dao { PurgeMapper purgeMapper = mapper(session); PurgeCommands purgeCommands = new PurgeCommands(session, profiler, system2); long start = System2.INSTANCE.now(); + boolean measuresMigrationEnabled = isMeasuresMigrationEnabled(); List branchUuids = session.getMapper(BranchMapper.class).selectByProjectUuid(uuid).stream() .map(BranchDto::getUuid) .filter(branchUuid -> !uuid.equals(branchUuid)) .toList(); - branchUuids.forEach(id -> deleteRootComponent(id, purgeMapper, purgeCommands)); + branchUuids.forEach(id -> deleteRootComponent(id, purgeMapper, purgeCommands, measuresMigrationEnabled)); - deleteRootComponent(uuid, purgeMapper, purgeCommands); + deleteRootComponent(uuid, purgeMapper, purgeCommands, measuresMigrationEnabled); auditPersister.deleteComponent(session, new ComponentNewValue(uuid, name, key, qualifier)); logProfiling(profiler, start); } @@ -218,7 +227,7 @@ public class PurgeDao implements Dao { LOG.info(""); } - private static void deleteRootComponent(String rootUuid, PurgeMapper mapper, PurgeCommands commands) { + private static void deleteRootComponent(String rootUuid, PurgeMapper mapper, PurgeCommands commands, boolean measuresMigrationEnabled) { List rootAndModulesOrSubviews = mapper.selectRootAndModulesOrSubviewsByProjectUuid(rootUuid); commands.deleteLinks(rootUuid); commands.deleteScannerCache(rootUuid); @@ -231,6 +240,9 @@ public class PurgeDao implements Dao { commands.deleteWebhooks(rootUuid); commands.deleteWebhookDeliveries(rootUuid); commands.deleteLiveMeasures(rootUuid); + if (measuresMigrationEnabled) { + commands.deleteJsonMeasures(rootUuid); + } commands.deleteProjectMappings(rootUuid); commands.deleteProjectAlmSettings(rootUuid); commands.deletePermissions(rootUuid); @@ -248,6 +260,13 @@ public class PurgeDao implements Dao { commands.deleteOutdatedProperties(rootUuid); } + private boolean isMeasuresMigrationEnabled() { + return Optional.ofNullable(propertiesDao.selectGlobalProperty(SYSTEM_MEASURES_MIGRATION_ENABLED)) + .map(PropertyDto::getValue) + .map(Boolean::valueOf) + .orElse(false); + } + /** * Delete the non root components (ie. sub-view, application or project copy) from the specified collection of {@link ComponentDto} * and data from their child tables. diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java index 182874c69e2..50f5473cc71 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -40,6 +40,8 @@ public interface PurgeMapper { Set selectDisabledComponentsWithLiveMeasures(@Param("branchUuid") String branchUuid); + Set selectDisabledComponentsWithJsonMeasures(@Param("branchUuid") String branchUuid); + void deleteAnalyses(@Param("analysisUuids") List analysisUuids); void deleteAnalysisProperties(@Param("analysisUuids") List analysisUuids); @@ -165,8 +167,12 @@ public interface PurgeMapper { void deleteLiveMeasuresByProjectUuid(@Param("projectUuid") String projectUuid); + void deleteJsonMeasuresByBranchUuid(@Param("branchUuid") String branchUuid); + void deleteLiveMeasuresByComponentUuids(@Param("componentUuids") List componentUuids); + void deleteJsonMeasuresByComponentUuids(@Param("componentUuids") List componentUuids); + void deleteNewCodePeriodsByRootUuid(String rootUuid); void deleteProjectAlmSettingsByProjectUuid(@Param("projectUuid") String projectUuid); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index 6105013fc92..1bba7511963 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -111,6 +111,16 @@ and p.branch_uuid=#{branchUuid,jdbcType=VARCHAR} + + delete from project_measures where @@ -585,11 +595,20 @@ delete from live_measures where project_uuid = #{projectUuid,jdbcType=VARCHAR} + + delete from measures where branch_uuid = #{branchUuid,jdbcType=VARCHAR} + + delete from live_measures where component_uuid in #{componentUuid, jdbcType=VARCHAR} + + delete from measures where component_uuid in #{componentUuid, jdbcType=VARCHAR} + + delete from user_dismissed_messages where project_uuid = #{projectUuid,jdbcType=VARCHAR} @@ -598,4 +617,3 @@ delete from scanner_analysis_cache where branch_uuid = #{branchUuid,jdbcType=VARCHAR} - diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java index 6cbc6b7517e..56e2e7fdc5b 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java @@ -22,6 +22,7 @@ package org.sonar.db.purge; import com.google.common.collect.ImmutableSet; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.sql.SQLException; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.Arrays; @@ -70,6 +71,7 @@ import org.sonar.db.event.EventDto; import org.sonar.db.event.EventTesting; import org.sonar.db.issue.IssueChangeDto; import org.sonar.db.issue.IssueDto; +import org.sonar.db.measure.JsonMeasureDto; import org.sonar.db.measure.LiveMeasureDto; import org.sonar.db.measure.MeasureDto; import org.sonar.db.metric.MetricDto; @@ -84,6 +86,7 @@ import org.sonar.db.user.UserDismissedMessageDto; import org.sonar.db.user.UserDto; import org.sonar.db.webhook.WebhookDeliveryLiteDto; import org.sonar.db.webhook.WebhookDto; +import org.sonar.server.platform.db.migration.adhoc.CreateMeasuresTable; import static com.google.common.base.MoreObjects.firstNonNull; import static java.nio.charset.StandardCharsets.UTF_8; @@ -98,6 +101,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import static org.sonar.core.config.CorePropertyDefinitions.SYSTEM_MEASURES_MIGRATION_ENABLED; import static org.sonar.db.ce.CeTaskTypes.REPORT; import static org.sonar.db.component.ComponentTesting.newBranchDto; import static org.sonar.db.component.ComponentTesting.newDirectory; @@ -267,7 +271,7 @@ public class PurgeDaoTest { } @Test - public void close_issues_clean_index_and_file_sources_of_disabled_components_specified_by_uuid_in_configuration() { + public void close_issues_clean_index_and_file_sources_of_disabled_components_specified_by_uuid_in_configuration() throws SQLException { RuleDto rule = db.rules().insert(); ComponentDto project = db.components().insertPublicProject(); db.components().insertSnapshot(project); @@ -306,6 +310,20 @@ public class PurgeDaoTest { LiveMeasureDto liveMeasureMetric1OnNonSelected = db.measures().insertLiveMeasure(enabledFile, metric1); LiveMeasureDto liveMeasureMetric2OnNonSelected = db.measures().insertLiveMeasure(enabledFile, metric2); assertThat(db.countRowsOfTable("live_measures")).isEqualTo(8); + + createMeasuresTable(); + db.properties().insertProperty(SYSTEM_MEASURES_MIGRATION_ENABLED, "true", null); + + db.measures().insertJsonMeasure(srcFile, + m -> m.addValue(metric1.getKey(), RandomUtils.nextInt(50)).addValue(metric2.getKey(), RandomUtils.nextInt(50))); + db.measures().insertJsonMeasure(dir, + m -> m.addValue(metric1.getKey(), RandomUtils.nextInt(50)).addValue(metric2.getKey(), RandomUtils.nextInt(50))); + db.measures().insertJsonMeasure(project, + m -> m.addValue(metric1.getKey(), RandomUtils.nextInt(50)).addValue(metric2.getKey(), RandomUtils.nextInt(50))); + db.measures().insertJsonMeasure(enabledFile, + m -> m.addValue(metric1.getKey(), RandomUtils.nextInt(50)).addValue(metric2.getKey(), RandomUtils.nextInt(50))); + assertThat(db.countRowsOfTable("measures")).isEqualTo(4); + PurgeListener purgeListener = mock(PurgeListener.class); // back to present @@ -348,6 +366,18 @@ public class PurgeDaoTest { assertThat(liveMeasureDtos) .extracting(LiveMeasureDto::getMetricUuid) .containsOnly(metric1.getUuid(), metric2.getUuid()); + + // delete json measures of selected + assertThat(db.countRowsOfTable("measures")).isEqualTo(2); + List measureDtos = Set.of(srcFile.uuid(), dir.uuid(), project.uuid(), enabledFile.uuid()).stream() + .map(component -> db.getDbClient().jsonMeasureDao().selectByComponentUuid(dbSession, component)) + .filter(Optional::isPresent).map(Optional::get).toList(); + assertThat(measureDtos) + .extracting(JsonMeasureDto::getComponentUuid) + .containsOnly(enabledFile.uuid(), project.uuid()); + assertThat(measureDtos) + .allSatisfy(dto -> assertThat(dto.getMetricValues()) + .containsOnlyKeys(metric1.getKey(), metric2.getKey())); } @Test @@ -1517,23 +1547,67 @@ public class PurgeDaoTest { } @Test - public void delete_live_measures_when_deleting_project() { + public void delete_live_measures_when_deleting_project() throws SQLException { + createMeasuresTable(); + db.properties().insertProperty(SYSTEM_MEASURES_MIGRATION_ENABLED, "true", null); + MetricDto metric = db.measures().insertMetric(); ComponentDto project1 = db.components().insertPublicProject(); ComponentDto module1 = db.components().insertComponent(ComponentTesting.newModuleDto(project1)); db.measures().insertLiveMeasure(project1, metric); db.measures().insertLiveMeasure(module1, metric); + db.measures().insertJsonMeasure(project1, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); + db.measures().insertJsonMeasure(module1, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); ComponentDto project2 = db.components().insertPublicProject(); ComponentDto module2 = db.components().insertComponent(ComponentTesting.newModuleDto(project2)); db.measures().insertLiveMeasure(project2, metric); db.measures().insertLiveMeasure(module2, metric); + db.measures().insertJsonMeasure(project2, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); + db.measures().insertJsonMeasure(module2, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); + + assertThat(db.countRowsOfTable("live_measures")).isEqualTo(4); + assertThat(db.countRowsOfTable("measures")).isEqualTo(4); underTest.deleteProject(dbSession, project1.uuid(), project1.qualifier(), project1.name(), project1.getKey()); + assertThat(db.countRowsOfTable("live_measures")).isEqualTo(2); + assertThat(db.countRowsOfTable("measures")).isEqualTo(2); assertThat(dbClient.liveMeasureDao().selectByComponentUuidsAndMetricUuids(dbSession, asList(project1.uuid(), module1.uuid()), asList(metric.getUuid()))).isEmpty(); assertThat(dbClient.liveMeasureDao().selectByComponentUuidsAndMetricUuids(dbSession, asList(project2.uuid(), module2.uuid()), asList(metric.getUuid()))).hasSize(2); + assertThat(dbClient.jsonMeasureDao().selectByComponentUuid(dbSession, project1.uuid())).isEmpty(); + assertThat(dbClient.jsonMeasureDao().selectByComponentUuid(dbSession, module1.uuid())).isEmpty(); + assertThat(dbClient.jsonMeasureDao().selectByComponentUuid(dbSession, project2.uuid())).isNotEmpty(); + assertThat(dbClient.jsonMeasureDao().selectByComponentUuid(dbSession, module2.uuid())).isNotEmpty(); + } + + @Test + public void do_not_delete_json_measures_when_migration_disabled() throws SQLException { + createMeasuresTable(); + db.properties().insertProperty(SYSTEM_MEASURES_MIGRATION_ENABLED, "false", null); + + MetricDto metric = db.measures().insertMetric(); + + ComponentDto project1 = db.components().insertPublicProject(); + ComponentDto module1 = db.components().insertComponent(ComponentTesting.newModuleDto(project1)); + db.measures().insertLiveMeasure(project1, metric); + db.measures().insertLiveMeasure(module1, metric); + db.measures().insertJsonMeasure(project1, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); + db.measures().insertJsonMeasure(module1, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); + + assertThat(db.countRowsOfTable("live_measures")).isEqualTo(2); + assertThat(db.countRowsOfTable("measures")).isEqualTo(2); + + underTest.deleteProject(dbSession, project1.uuid(), project1.qualifier(), project1.name(), project1.getKey()); + + assertThat(db.countRowsOfTable("live_measures")).isZero(); + assertThat(db.countRowsOfTable("measures")).isEqualTo(2); + } + + private void createMeasuresTable() throws SQLException { + new CreateMeasuresTable(db.getDbClient().getDatabase()).execute(); + db.executeDdl("truncate table measures"); } private void verifyNoEffect(ComponentDto firstRoot, ComponentDto... otherRoots) { diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoWithAuditTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoWithAuditTest.java index f41c48d76e2..ccedabf1c16 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoWithAuditTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoWithAuditTest.java @@ -31,6 +31,7 @@ import org.sonar.db.DbTester; import org.sonar.db.audit.AuditPersister; import org.sonar.db.audit.model.ComponentNewValue; import org.sonar.db.component.ComponentDto; +import org.sonar.db.property.PropertiesDao; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -50,7 +51,7 @@ public class PurgeDaoWithAuditTest { private final DbSession dbSession = db.getSession(); private final AuditPersister auditPersister = mock(AuditPersister.class); - private final PurgeDao underTestWithPersister = new PurgeDao(system2, auditPersister); + private final PurgeDao underTestWithPersister = new PurgeDao(system2, auditPersister, mock(PropertiesDao.class)); @Test public void delete_project_persist_audit_with_uuid_and_name() { diff --git a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureDbTester.java b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureDbTester.java index a036b7a104d..49e0b347257 100644 --- a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureDbTester.java +++ b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureDbTester.java @@ -78,6 +78,18 @@ public class MeasureDbTester { return dto; } + @SafeVarargs + public final JsonMeasureDto insertJsonMeasure(ComponentDto component, Consumer... consumers) { + JsonMeasureDto dto = new JsonMeasureDto() + .setComponentUuid(component.uuid()) + .setBranchUuid(component.branchUuid()); + Arrays.stream(consumers).forEach(c -> c.accept(dto)); + dto.computeJsonValueHash(); + dbClient.jsonMeasureDao().insert(db.getSession(), dto); + db.commit(); + return dto; + } + @SafeVarargs public final MetricDto insertMetric(Consumer... consumers) { MetricDto metricDto = newMetricDto(); -- 2.39.5