From 9782e559751ea8fb8d4a0100180fff1877aac9f9 Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Thu, 13 Dec 2018 16:05:39 -0600 Subject: [PATCH] SONAR-11585 Purge disabled components --- .../org/sonar/db/purge/PurgeCommands.java | 23 ++++++++ .../java/org/sonar/db/purge/PurgeDao.java | 12 +++-- .../java/org/sonar/db/purge/PurgeMapper.java | 2 + .../org/sonar/db/purge/PurgeMapper.xml | 11 ++++ .../java/org/sonar/db/purge/PurgeDaoTest.java | 53 ++++++++++++++----- 5 files changed, 84 insertions(+), 17 deletions(-) diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java index d15c3d3d818..23908f55d96 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -202,6 +202,29 @@ class PurgeCommands { profiler.stop(); } + void deleteDisabledComponentsWithoutIssues(List disabledComponentsWithoutIssue) { + if (disabledComponentsWithoutIssue.isEmpty()) { + return; + } + List> idPartitions = Lists.partition(IdUuidPairs.ids(disabledComponentsWithoutIssue), MAX_RESOURCES_PER_QUERY); + List> uuidsPartitions = Lists.partition(IdUuidPairs.uuids(disabledComponentsWithoutIssue), MAX_RESOURCES_PER_QUERY); + + profiler.start("deleteDisabledComponentsWithoutIssues (properties)"); + idPartitions.forEach(purgeMapper::deletePropertiesByComponentIds); + session.commit(); + profiler.stop(); + + profiler.start("deleteDisabledComponentsWithoutIssues (manual_measures)"); + uuidsPartitions.forEach(purgeMapper::deleteManualMeasuresByComponentUuids); + session.commit(); + profiler.stop(); + + profiler.start("deleteDisabledComponentsWithoutIssues (projects)"); + uuidsPartitions.forEach(purgeMapper::deleteComponentsByUuids); + session.commit(); + profiler.stop(); + } + void deleteComponents(String rootUuid) { profiler.start("deleteComponents (projects)"); purgeMapper.deleteComponentsByProjectUuid(rootUuid); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java index 12d3c0f8cb4..1ffbaa41847 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -66,8 +66,10 @@ public class PurgeDao implements Dao { deleteAbortedAnalyses(rootUuid, commands); deleteDataOfComponentsWithoutHistoricalData(session, rootUuid, conf.getScopesWithoutHistoricalData(), commands); purgeAnalyses(commands, rootUuid); - purgeDisabledComponents(session, conf, listener); + purgeDisabledComponents(session, mapper, conf, listener); deleteOldClosedIssues(conf, mapper, listener); + + deleteOldDisabledComponents(commands, mapper, rootUuid); purgeStaleBranches(commands, conf, mapper, rootUuid); } @@ -142,8 +144,7 @@ public class PurgeDao implements Dao { purgeCommands.deleteComponentMeasures(analysisUuids, componentWithoutHistoricalDataUuids); } - private void purgeDisabledComponents(DbSession session, PurgeConfiguration conf, PurgeListener listener) { - PurgeMapper mapper = mapper(session); + private void purgeDisabledComponents(DbSession session, PurgeMapper mapper, PurgeConfiguration conf, PurgeListener listener) { executeLargeInputs(conf.getDisabledComponentUuids(), input -> { mapper.deleteFileSourcesByFileUuid(input); @@ -157,6 +158,11 @@ public class PurgeDao implements Dao { session.commit(); } + private static void deleteOldDisabledComponents(PurgeCommands commands, PurgeMapper mapper, String rootUuid) { + List disabledComponentsWithoutIssue = mapper.selectDisabledComponentsWithoutIssues(rootUuid); + commands.deleteDisabledComponentsWithoutIssues(disabledComponentsWithoutIssue); + } + public List selectPurgeableAnalyses(String componentUuid, DbSession session) { PurgeMapper mapper = mapper(session); Stream allPurgeableAnalyses = Stream.concat( diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java index 28fabd2dcbe..e1245f70c92 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -94,6 +94,8 @@ public interface PurgeMapper { @CheckForNull String selectManualBaseline(@Param("projectUuid") String projectUuid); + List selectDisabledComponentsWithoutIssues(@Param("projectUuid") String projectUuid); + void deleteIssuesFromKeys(@Param("keys") List keys); void deleteIssueChangesFromIssueKeys(@Param("issueKeys") List issueKeys); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index 8fe582263e0..f5c14063d8f 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -317,6 +317,17 @@ + + DELETE FROM issues WHERE kee IN diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java index d9fbfce6e4f..393dd2910f7 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java @@ -36,6 +36,7 @@ import org.apache.commons.lang.time.DateUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.sonar.api.issue.Issue; import org.sonar.api.resources.Scopes; import org.sonar.api.utils.System2; import org.sonar.core.util.CloseableIterator; @@ -144,9 +145,7 @@ public class PurgeDaoTest { db.issues().insert(rule, shortBranch, subModule); db.issues().insert(rule, shortBranch, module); - // back to present - when(system2.now()).thenReturn(new Date().getTime()); - underTest.purge(dbSession, newConfigurationWith30Days(system2, project.uuid(), project.uuid()), PurgeListener.EMPTY, new PurgeProfiler()); + underTest.purge(dbSession, newConfigurationWith30Days(System2.INSTANCE, project.uuid(), project.uuid()), PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); assertThat(uuidsIn("projects")).containsOnly(project.uuid(), longBranch.uuid(), recentShortBranch.uuid()); @@ -170,9 +169,7 @@ public class PurgeDaoTest { db.issues().insert(rule, pullRequest, subModule); db.issues().insert(rule, pullRequest, module); - // back to present - when(system2.now()).thenReturn(new Date().getTime()); - underTest.purge(dbSession, newConfigurationWith30Days(system2, project.uuid(), project.uuid()), PurgeListener.EMPTY, new PurgeProfiler()); + underTest.purge(dbSession, newConfigurationWith30Days(System2.INSTANCE, project.uuid(), project.uuid()), PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); assertThat(uuidsIn("projects")).containsOnly(project.uuid(), longBranch.uuid(), recentPullRequest.uuid()); @@ -219,10 +216,8 @@ public class PurgeDaoTest { @Test public void close_issues_clean_index_and_file_sources_of_disabled_components_specified_by_uuid_in_configuration() { - // components and issues, updated 31 days ago - when(system2.now()).thenReturn(DateUtils.addDays(new Date(), -31).getTime()); RuleDefinitionDto rule = db.rules().insert(); - ComponentDto project = db.components().insertMainBranch(p -> p.setEnabled(false)); + ComponentDto project = db.components().insertMainBranch(); db.components().insertSnapshot(project); db.components().insertSnapshot(project); db.components().insertSnapshot(project, s -> s.setLast(false)); @@ -261,7 +256,6 @@ public class PurgeDaoTest { assertThat(db.countRowsOfTable("live_measures")).isEqualTo(8); // back to present - when(system2.now()).thenReturn(new Date().getTime()); underTest.purge(dbSession, newConfigurationWith30Days(system2, project.uuid(), project.uuid(), module.uuid(), dir.uuid(), srcFile.uuid(), testFile.uuid()), PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); @@ -272,8 +266,8 @@ public class PurgeDaoTest { // close open issues of selected assertThat(db.countSql("select count(*) from issues where status = 'CLOSED'")).isEqualTo(4); for (IssueDto issue : Arrays.asList(openOnFile, confirmOnFile, openOnDir, confirmOnDir)) { - assertThat(db.getDbClient().issueDao().selectByKey(dbSession, issue.getKey()).get()) - .extracting("status", "resolution") + assertThat(db.getDbClient().issueDao().selectOrFailByKey(dbSession, issue.getKey())) + .extracting(IssueDto::getStatus, IssueDto::getResolution) .containsExactlyInAnyOrder("CLOSED", "REMOVED"); } for (IssueDto issue : Arrays.asList(openOnNonSelected, confirmOnNonSelected)) { @@ -288,8 +282,9 @@ public class PurgeDaoTest { // deletes live measure of selected assertThat(db.countRowsOfTable("live_measures")).isEqualTo(4); - List liveMeasureDtos = db.getDbClient().liveMeasureDao().selectByComponentUuidsAndMetricIds(dbSession, - ImmutableSet.of(srcFile.uuid(), dir.uuid(), project.uuid(), nonSelectedFile.uuid()), ImmutableSet.of(metric1.getId(), metric2.getId())); + List liveMeasureDtos = db.getDbClient().liveMeasureDao() + .selectByComponentUuidsAndMetricIds(dbSession, ImmutableSet.of(srcFile.uuid(), dir.uuid(), project.uuid(), nonSelectedFile.uuid()), + ImmutableSet.of(metric1.getId(), metric2.getId())); assertThat(liveMeasureDtos) .extracting(LiveMeasureDto::getComponentUuid) .containsOnly(nonSelectedFile.uuid(), project.uuid()); @@ -999,6 +994,36 @@ public class PurgeDaoTest { assertThat(db.getDbClient().issueDao().selectByKey(dbSession, notClosed.getKey())).isNotEmpty(); } + @Test + public void delete_disabled_components_without_issues() { + ComponentDto project = db.components().insertMainBranch(p -> p.setEnabled(true)); + ComponentDto enabledFileWithIssues = db.components().insertComponent(newFileDto(project).setEnabled(true)); + ComponentDto disabledFileWithIssues = db.components().insertComponent(newFileDto(project).setEnabled(false)); + ComponentDto enabledFileWithoutIssues = db.components().insertComponent(newFileDto(project).setEnabled(true)); + ComponentDto disabledFileWithoutIssues = db.components().insertComponent(newFileDto(project).setEnabled(false)); + + RuleDefinitionDto rule = db.rules().insert(); + IssueDto closed1 = db.issues().insert(rule, project, enabledFileWithIssues, issue -> { + issue.setStatus("CLOSED"); + issue.setResolution(Issue.RESOLUTION_FIXED); + issue.setIssueCloseDate(new Date()); + }); + IssueDto closed2 = db.issues().insert(rule, project, disabledFileWithIssues, issue -> { + issue.setStatus("CLOSED"); + issue.setResolution(Issue.RESOLUTION_FIXED); + issue.setIssueCloseDate(new Date()); + }); + + underTest.purge(dbSession, + newConfigurationWith30Days(System2.INSTANCE, project.uuid(), disabledFileWithIssues.uuid(), disabledFileWithoutIssues.uuid()), + PurgeListener.EMPTY, new PurgeProfiler()); + + assertThat(db.getDbClient().componentDao().selectByProjectUuid(project.uuid(), dbSession)) + .extracting("uuid") + .containsOnly(project.uuid(), enabledFileWithIssues.uuid(), disabledFileWithIssues.uuid(), + enabledFileWithoutIssues.uuid()); + } + @Test public void deleteProject_deletes_webhook_deliveries() { ComponentDto project = db.components().insertPublicProject(); -- 2.39.5