From: Eric Giffon Date: Fri, 30 Aug 2024 14:21:31 +0000 (+0200) Subject: SONAR-22872 Purge measures table X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=75a2e6255fb114c56b0b3d1273ebec59c374714a;p=sonarqube.git SONAR-22872 Purge measures table --- diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeDaoIT.java index 489aaf8b2e8..59c3dea76c3 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeDaoIT.java @@ -38,6 +38,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.lang3.time.DateUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -78,6 +79,7 @@ import org.sonar.db.issue.AnticipatedTransitionDto; import org.sonar.db.issue.IssueChangeDto; import org.sonar.db.issue.IssueDto; import org.sonar.db.measure.LiveMeasureDto; +import org.sonar.db.measure.MeasureDto; import org.sonar.db.measure.ProjectMeasureDto; import org.sonar.db.metric.MetricDto; import org.sonar.db.newcodeperiod.NewCodePeriodDto; @@ -331,12 +333,21 @@ project.getProjectDto().getUuid()), PurgeListener.EMPTY, new PurgeProfiler()); LiveMeasureDto liveMeasureMetric1OnNonSelected = db.measures().insertLiveMeasure(enabledFile, metric1); LiveMeasureDto liveMeasureMetric2OnNonSelected = db.measures().insertLiveMeasure(enabledFile, metric2); assertThat(db.countRowsOfTable("live_measures")).isEqualTo(8); - PurgeListener purgeListener = mock(PurgeListener.class); + + db.measures().insertMeasure(srcFile, + m -> m.addValue(metric1.getKey(), RandomUtils.nextInt(50)).addValue(metric2.getKey(), RandomUtils.nextInt(50))); + db.measures().insertMeasure(dir, + m -> m.addValue(metric1.getKey(), RandomUtils.nextInt(50)).addValue(metric2.getKey(), RandomUtils.nextInt(50))); + db.measures().insertMeasure(mainBranch, + m -> m.addValue(metric1.getKey(), RandomUtils.nextInt(50)).addValue(metric2.getKey(), RandomUtils.nextInt(50))); + db.measures().insertMeasure(enabledFile, + m -> m.addValue(metric1.getKey(), RandomUtils.nextInt(50)).addValue(metric2.getKey(), RandomUtils.nextInt(50))); + assertThat(db.countRowsOfTable("measures")).isEqualTo(4); // back to present - Set selectedComponentUuids = ImmutableSet.of(srcFile.uuid(), testFile.uuid()); + Set selectedComponentUuids = Set.of(srcFile.uuid(), testFile.uuid()); underTest.purge(dbSession, newConfigurationWith30Days(system2, mainBranch.uuid(), projectData.projectUuid(), selectedComponentUuids), - purgeListener, new PurgeProfiler()); + mock(PurgeListener.class), new PurgeProfiler()); dbSession.commit(); // set purged=true for non-last snapshot @@ -362,14 +373,26 @@ project.getProjectDto().getUuid()), PurgeListener.EMPTY, new PurgeProfiler()); // deletes live measure of selected assertThat(db.countRowsOfTable("live_measures")).isEqualTo(4); List liveMeasureDtos = db.getDbClient().liveMeasureDao() - .selectByComponentUuidsAndMetricUuids(dbSession, ImmutableSet.of(srcFile.uuid(), dir.uuid(), mainBranch.uuid(), enabledFile.uuid()), - ImmutableSet.of(metric1.getUuid(), metric2.getUuid())); + .selectByComponentUuidsAndMetricUuids(dbSession, Set.of(srcFile.uuid(), dir.uuid(), mainBranch.uuid(), enabledFile.uuid()), + Set.of(metric1.getUuid(), metric2.getUuid())); assertThat(liveMeasureDtos) .extracting(LiveMeasureDto::getComponentUuid) .containsOnly(enabledFile.uuid(), mainBranch.uuid()); assertThat(liveMeasureDtos) .extracting(LiveMeasureDto::getMetricUuid) .containsOnly(metric1.getUuid(), metric2.getUuid()); + + // delete measures of selected + assertThat(db.countRowsOfTable("measures")).isEqualTo(2); + List measureDtos = Set.of(srcFile.uuid(), dir.uuid(), mainBranch.uuid(), enabledFile.uuid()).stream() + .map(component -> db.getDbClient().measureDao().selectMeasure(dbSession, component)) + .filter(Optional::isPresent).map(Optional::get).toList(); + assertThat(measureDtos) + .extracting(MeasureDto::getComponentUuid) + .containsOnly(enabledFile.uuid(), mainBranch.uuid()); + assertThat(measureDtos) + .allSatisfy(dto -> assertThat(dto.getMetricValues()) + .containsOnlyKeys(metric1.getKey(), metric2.getKey())); } @Test @@ -1698,18 +1721,26 @@ project.getProjectDto().getKey()); ComponentDto dir1 = db.components().insertComponent(newDirectory(project1, "path")); db.measures().insertLiveMeasure(project1, metric); db.measures().insertLiveMeasure(dir1, metric); + db.measures().insertMeasure(project1, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); + db.measures().insertMeasure(dir1, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); ComponentDto project2 = db.components().insertPublicProject().getMainBranchComponent(); ComponentDto dir2 = db.components().insertComponent(newDirectory(project2, "path")); db.measures().insertLiveMeasure(project2, metric); db.measures().insertLiveMeasure(dir2, metric); + db.measures().insertMeasure(project2, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); + db.measures().insertMeasure(dir2, m -> m.addValue(metric.getKey(), RandomUtils.nextInt(50))); underTest.deleteProject(dbSession, project1.uuid(), project1.qualifier(), project1.name(), project1.getKey()); - assertThat(dbClient.liveMeasureDao().selectByComponentUuidsAndMetricUuids(dbSession, asList(project1.uuid(), dir1.uuid()), - asList(metric.getUuid()))).isEmpty(); - assertThat(dbClient.liveMeasureDao().selectByComponentUuidsAndMetricUuids(dbSession, asList(project2.uuid(), dir2.uuid()), - asList(metric.getUuid()))).hasSize(2); + assertThat(dbClient.liveMeasureDao().selectByComponentUuidsAndMetricUuids(dbSession, List.of(project1.uuid(), dir1.uuid()), + List.of(metric.getUuid()))).isEmpty(); + assertThat(dbClient.liveMeasureDao().selectByComponentUuidsAndMetricUuids(dbSession, List.of(project2.uuid(), dir2.uuid()), + List.of(metric.getUuid()))).hasSize(2); + assertThat(dbClient.measureDao().selectMeasure(dbSession, project1.uuid())).isEmpty(); + assertThat(dbClient.measureDao().selectMeasure(dbSession, dir1.uuid())).isEmpty(); + assertThat(dbClient.measureDao().selectMeasure(dbSession, project2.uuid())).isNotEmpty(); + assertThat(dbClient.measureDao().selectMeasure(dbSession, dir2.uuid())).isNotEmpty(); } private void verifyNoEffect(ComponentDto firstRoot, ComponentDto... otherRoots) { @@ -1826,15 +1857,15 @@ projects[2].getMainBranchComponent().uuid(), ComponentDto subview = db.components().insertComponent(newSubPortfolio(view)); ComponentDto pc = db.components().insertComponent(newProjectCopy("a", db.components().insertPrivateProject().getMainBranchComponent() , view)); - insertMeasureFor(view, subview, pc); - assertThat(getComponentUuidsOfMeasures()).containsOnly(view.uuid(), subview.uuid(), pc.uuid()); + insertProjectMeasureFor(view, subview, pc); + assertThat(getComponentUuidsOfProjectMeasures()).containsOnly(view.uuid(), subview.uuid(), pc.uuid()); underTest.deleteNonRootComponentsInView(dbSession, singletonList(pc)); - assertThat(getComponentUuidsOfMeasures()) + assertThat(getComponentUuidsOfProjectMeasures()) .containsOnly(view.uuid(), subview.uuid()); underTest.deleteNonRootComponentsInView(dbSession, singletonList(subview)); - assertThat(getComponentUuidsOfMeasures()) + assertThat(getComponentUuidsOfProjectMeasures()) .containsOnly(view.uuid()); } @@ -2021,12 +2052,12 @@ oldCreationDate)); null, project.getKey(), null, null); } - private Stream getComponentUuidsOfMeasures() { + private Stream getComponentUuidsOfProjectMeasures() { return db.select("select component_uuid as \"COMPONENT_UUID\" from project_measures").stream() .map(row -> (String) row.get("COMPONENT_UUID")); } - private void insertMeasureFor(ComponentDto... components) { + private void insertProjectMeasureFor(ComponentDto... components) { Arrays.stream(components).forEach(componentDto -> db.getDbClient().projectMeasureDao().insert(dbSession, new ProjectMeasureDto() .setMetricUuid(randomAlphabetic(3)) .setComponentUuid(componentDto.uuid()) diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java index 40a6b39d848..68f55067421 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -167,6 +167,15 @@ class PurgeCommands { }); profiler.stop(); + profiler.start("purgeDisabledComponents (measures)"); + executeLargeInputs( + purgeMapper.selectDisabledComponentsWithMeasures(rootComponentUuid), + input -> { + purgeMapper.deleteMeasuresByComponentUuids(input); + return input; + }); + profiler.stop(); + session.commit(); } @@ -456,6 +465,13 @@ class PurgeCommands { profiler.stop(); } + void deleteMeasures(String rootUuid) { + profiler.start("deleteMeasures (measures)"); + purgeMapper.deleteMeasuresByBranchUuid(rootUuid); + session.commit(); + profiler.stop(); + } + void deleteNewCodePeriodsForProject(String projectUuid) { profiler.start("deleteNewCodePeriods (new_code_periods)"); purgeMapper.deleteNewCodePeriodsByProjectUuid(projectUuid); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java index 4ed7bc57e07..51e06e94f90 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -272,6 +272,7 @@ public class PurgeDao implements Dao { commands.deleteCeActivity(branchUuid); commands.deleteCeQueue(branchUuid); commands.deleteLiveMeasures(branchUuid); + commands.deleteMeasures(branchUuid); commands.deleteNewCodePeriodsForBranch(branchUuid); commands.deleteBranch(branchUuid); commands.deleteApplicationBranchProjects(branchUuid); @@ -295,6 +296,7 @@ public class PurgeDao implements Dao { commands.deleteWebhooks(projectUuid); commands.deleteWebhookDeliveries(projectUuid); commands.deleteLiveMeasures(projectUuid); + commands.deleteMeasures(projectUuid); commands.deleteProjectAlmSettings(projectUuid); commands.deletePermissions(projectUuid); commands.deleteNewCodePeriodsForProject(projectUuid); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java index 4b916638fcf..6053dc0fe32 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -40,6 +40,8 @@ public interface PurgeMapper { Set selectDisabledComponentsWithLiveMeasures(@Param("branchUuid") String branchUuid); + Set selectDisabledComponentsWithMeasures(@Param("branchUuid") String branchUuid); + void deleteAnalyses(@Param("analysisUuids") List analysisUuids); void deleteAnalysisProperties(@Param("analysisUuids") List analysisUuids); @@ -169,8 +171,12 @@ public interface PurgeMapper { void deleteLiveMeasuresByProjectUuid(@Param("projectUuid") String projectUuid); + void deleteMeasuresByBranchUuid(@Param("branchUuid") String branchUuid); + void deleteLiveMeasuresByComponentUuids(@Param("componentUuids") List componentUuids); + void deleteMeasuresByComponentUuids(@Param("componentUuids") List componentUuids); + void deleteNewCodePeriodsByProjectUuid(String projectUuid); void deleteNewCodePeriodsByBranchUuid(String branchUuid); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index 7aa1312f799..7e4ebdafb46 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -124,6 +124,16 @@ and p.branch_uuid=#{branchUuid,jdbcType=VARCHAR} + + delete from project_measures where @@ -631,11 +641,20 @@ delete from live_measures where project_uuid = #{projectUuid,jdbcType=VARCHAR} + + delete from measures where branch_uuid = #{branchUuid,jdbcType=VARCHAR} + + delete from live_measures where component_uuid in #{componentUuid, jdbcType=VARCHAR} + + delete from measures where component_uuid in #{componentUuid, jdbcType=VARCHAR} + + delete from user_dismissed_messages where project_uuid = #{projectUuid,jdbcType=VARCHAR} @@ -670,4 +689,3 @@ delete from issues_fixed where pull_request_uuid = #{branchUuid,jdbcType=VARCHAR} - diff --git a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureDbTester.java b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureDbTester.java index 4cff115a957..5c2f3e2eaf6 100644 --- a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureDbTester.java +++ b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureDbTester.java @@ -28,7 +28,6 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import org.sonar.api.measures.CoreMetrics; import org.sonar.db.DbClient; -import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.component.BranchDto; import org.sonar.db.component.ComponentDto; @@ -108,6 +107,17 @@ public class MeasureDbTester { return insertLiveMeasure(projectData.getMainBranchComponent(), metric, consumers); } + @SafeVarargs + public final MeasureDto insertMeasure(ComponentDto component, Consumer... consumers) { + MeasureDto dto = new MeasureDto() + .setComponentUuid(component.uuid()) + .setBranchUuid(component.branchUuid()); + Arrays.stream(consumers).forEach(c -> c.accept(dto)); + dto.computeJsonValueHash(); + dbClient.measureDao().insert(db.getSession(), dto); + db.getSession().commit(); + return dto; + } @SafeVarargs public final MetricDto insertMetric(Consumer... consumers) {