From: Matteo Mara Date: Wed, 26 Jul 2023 14:49:06 +0000 (+0200) Subject: SONAR-20058 purge old anticipated transitions according to a max age property X-Git-Tag: 10.2.0.77647~259 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=17716a7f1f2a0847bb6adbf6d8d8451878cef2ab;p=sonarqube.git SONAR-20058 purge old anticipated transitions according to a max age property --- diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeCommandsIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeCommandsIT.java index 4778f1eb75d..923955f5147 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeCommandsIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeCommandsIT.java @@ -22,6 +22,8 @@ package org.sonar.db.purge; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -47,6 +49,7 @@ import org.sonar.db.component.SnapshotDto; import org.sonar.db.dialect.Dialect; import org.sonar.db.duplication.DuplicationUnitDto; import org.sonar.db.entity.EntityDto; +import org.sonar.db.issue.AnticipatedTransitionDto; import org.sonar.db.issue.IssueDto; import org.sonar.db.metric.MetricDto; import org.sonar.db.newcodeperiod.NewCodePeriodType; @@ -779,6 +782,28 @@ public class PurgeCommandsIT { .extracting(r -> r.getUuid()).containsExactly("uuid2"); } + @Test + public void deleteAnticipatedTransitions_shouldDeleteAnticipatedTransitionsOlderThanDate() { + ComponentDto projectDto = ComponentTesting.newPrivateProjectDto(); + //dates at the boundary of the deletion threshold set to 30 days + Instant okCreationDate = Instant.now().minus(29, ChronoUnit.DAYS); + Instant oldCreationDate = Instant.now().minus(31, ChronoUnit.DAYS); + + dbTester.getDbClient().anticipatedTransitionDao().insert(dbTester.getSession(), getAnticipatedTransitionsDto(projectDto.uuid() + "okTransition", projectDto.uuid(), okCreationDate)); + dbTester.getDbClient().anticipatedTransitionDao().insert(dbTester.getSession(), getAnticipatedTransitionsDto(projectDto.uuid() + "oldTransition", projectDto.uuid(), oldCreationDate)); + + underTest.deleteAnticipatedTransitions(projectDto.uuid(), Instant.now().minus(30, ChronoUnit.DAYS).toEpochMilli()); + + List anticipatedTransitionDtos = dbTester.getDbClient().anticipatedTransitionDao().selectByProjectUuid(dbTester.getSession(), projectDto.uuid()); + + assertThat(anticipatedTransitionDtos).hasSize(1); + assertThat(anticipatedTransitionDtos.get(0).getUuid()).isEqualTo(projectDto.uuid() + "okTransition"); + } + + private AnticipatedTransitionDto getAnticipatedTransitionsDto(String uuid, String projectUuid, Instant creationDate) { + return new AnticipatedTransitionDto(uuid, projectUuid, "userUuid", "transition", null, null, null, null, "rule:key", "filePath", creationDate.toEpochMilli()); + } + private void addPermissions(EntityDto projectDto) { if (!projectDto.isPrivate()) { dbTester.users().insertEntityPermissionOnAnyone("foo1", projectDto); diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeDaoIT.java index b8182de0987..1cf200413ee 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/purge/PurgeDaoIT.java @@ -75,6 +75,7 @@ import org.sonar.db.dialect.Dialect; import org.sonar.db.event.EventComponentChangeDto; import org.sonar.db.event.EventDto; import org.sonar.db.event.EventTesting; +import org.sonar.db.issue.AnticipatedTransitionDto; import org.sonar.db.issue.IssueChangeDto; import org.sonar.db.issue.IssueDto; import org.sonar.db.measure.LiveMeasureDto; @@ -1834,6 +1835,31 @@ public class PurgeDaoIT { assertThat(scannerContextExists("RECENT")).isTrue(); } + @Test + public void purge_old_anticipated_transitions() { + ProjectData project = db.components().insertPrivateProject(); + + //dates at the boundary of the deletion threshold set to 30 days + Date okCreationDate = DateUtils.addDays(new Date(), -29); + Date oldCreationDate = DateUtils.addDays(new Date(), -31); + + dbClient.anticipatedTransitionDao().insert(dbSession, getAnticipatedTransitionsDto("okTransition", project.projectUuid(), okCreationDate)); + dbClient.anticipatedTransitionDao().insert(dbSession, getAnticipatedTransitionsDto("oldTransition", project.projectUuid(), oldCreationDate)); + + underTest.purge(dbSession, newConfigurationWith30Days(System2.INSTANCE, project.getMainBranchComponent().uuid(), project.projectUuid()), PurgeListener.EMPTY, + new PurgeProfiler()); + dbSession.commit(); + + List anticipatedTransitionDtos = dbClient.anticipatedTransitionDao().selectByProjectUuid(dbSession, project.projectUuid()); + + assertThat(anticipatedTransitionDtos).hasSize(1); + assertThat(anticipatedTransitionDtos.get(0).getUuid()).isEqualTo("okTransition"); + } + + private AnticipatedTransitionDto getAnticipatedTransitionsDto(String uuid, String projectUuid, Date creationDate) { + return new AnticipatedTransitionDto(uuid, projectUuid, "userUuid", "transition", null, null, null, null, "rule:key", "filepath", creationDate.getTime()); + } + private Optional selectActivity(String taskUuid) { return db.getDbClient().ceActivityDao().selectByUuid(db.getSession(), taskUuid); } @@ -2011,7 +2037,7 @@ public class PurgeDaoIT { } private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String rootUuid, String projectUuid, Set disabledComponentUuids) { - return new PurgeConfiguration(rootUuid, projectUuid, 30, Optional.of(30), system2, disabledComponentUuids); + return new PurgeConfiguration(rootUuid, projectUuid, 30, Optional.of(30), system2, disabledComponentUuids, 30); } private Stream uuidsOfAnalysesOfRoot(ComponentDto rootComponent) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java index fca4f6b9805..610bb1868c0 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -489,7 +489,6 @@ class PurgeCommands { profiler.stop(); } - public void deleteReportSubscriptions(String rootUuid) { profiler.start("deleteReportSubscriptions (report_subscriptions)"); purgeMapper.deleteReportSubscriptionsByBranchUuid(rootUuid); @@ -497,5 +496,10 @@ class PurgeCommands { profiler.stop(); } - + public void deleteAnticipatedTransitions(String projectUuid, long createdAt) { + profiler.start("deleteAnticipatedTransitions (anticipated_transitions)"); + purgeMapper.deleteAnticipatedTransitionsByProjectUuidAndCreationDate(projectUuid, createdAt); + session.commit(); + profiler.stop(); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java index 786ae8b349c..e6ccc976644 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java @@ -20,6 +20,8 @@ package org.sonar.db.purge; import com.google.common.annotations.VisibleForTesting; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Optional; import java.util.Set; @@ -38,19 +40,23 @@ public class PurgeConfiguration { private final System2 system2; private final Set disabledComponentUuids; + private final int maxAgeInDaysOfAnticipatedTransitions; + public PurgeConfiguration(String rootUuid, String projectUuid, int maxAgeInDaysOfClosedIssues, - Optional maxAgeInDaysOfInactiveBranches, System2 system2, Set disabledComponentUuids) { + Optional maxAgeInDaysOfInactiveBranches, System2 system2, Set disabledComponentUuids, int maxAgeInDaysOfAnticipatedTransitions) { this.rootUuid = rootUuid; this.projectUuid = projectUuid; this.maxAgeInDaysOfClosedIssues = maxAgeInDaysOfClosedIssues; this.system2 = system2; this.disabledComponentUuids = disabledComponentUuids; this.maxAgeInDaysOfInactiveBranches = maxAgeInDaysOfInactiveBranches; + this.maxAgeInDaysOfAnticipatedTransitions = maxAgeInDaysOfAnticipatedTransitions; } public static PurgeConfiguration newDefaultPurgeConfiguration(Configuration config, String rootUuid, String projectUuid, Set disabledComponentUuids) { return new PurgeConfiguration(rootUuid, projectUuid, config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES).get(), - config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_INACTIVE_BRANCHES_AND_PRS), System2.INSTANCE, disabledComponentUuids); + config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_INACTIVE_BRANCHES_AND_PRS), System2.INSTANCE, disabledComponentUuids, + config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_ANTICIPATED_TRANSITIONS).get()); } /** @@ -82,6 +88,11 @@ public class PurgeConfiguration { return maxAgeInDaysOfInactiveBranches.map(age -> DateUtils.addDays(new Date(system2.now()), -age)); } + public Instant maxLiveDateOfAnticipatedTransitions() { + return Instant.ofEpochMilli(system2.now()).minus(maxAgeInDaysOfAnticipatedTransitions, ChronoUnit.DAYS); + } + + @VisibleForTesting @CheckForNull Date maxLiveDateOfClosedIssues(Date now) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java index 0e35b7cd53f..9ceba885c7a 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -71,9 +71,11 @@ public class PurgeDao implements Dao { deleteOrphanIssues(mapper, rootUuid); purgeOldCeActivities(session, rootUuid, commands); purgeOldCeScannerContexts(session, rootUuid, commands); + deleteOldAnticipatedTransitions(commands, conf, conf.projectUuid()); deleteOldDisabledComponents(commands, mapper, rootUuid); purgeStaleBranches(commands, conf, mapper, rootUuid); + } private static void purgeStaleBranches(PurgeCommands commands, PurgeConfiguration conf, PurgeMapper mapper, String rootUuid) { @@ -113,6 +115,11 @@ public class PurgeDao implements Dao { deleteIssues(mapper, issueKeys); } + private static void deleteOldAnticipatedTransitions(PurgeCommands commands, PurgeConfiguration purgeConfiguration, String projectUuid) { + LOG.debug("<- Delete Old Anticipated Transitions"); + commands.deleteAnticipatedTransitions(projectUuid, purgeConfiguration.maxLiveDateOfAnticipatedTransitions().toEpochMilli()); + } + private static void deleteOldClosedIssues(PurgeConfiguration conf, PurgeMapper mapper, PurgeListener listener) { Date toDate = conf.maxLiveDateOfClosedIssues(); String rootUuid = conf.rootUuid(); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java index 019033026ee..44651d1e872 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -184,4 +184,6 @@ public interface PurgeMapper { void deleteReportSchedulesByPortfolioUuids(@Param("portfolioUuids") List portfolioUuids); void deleteReportSubscriptionsByPortfolioUuids(@Param("portfolioUuids") List portfolioUuids); + + void deleteAnticipatedTransitionsByProjectUuidAndCreationDate(@Param("projectUuid") String projectUuid, @Param("createdAtBefore") Long createdAtBefore); } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index 770955921ca..1e7702bde22 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -616,12 +616,17 @@ delete from report_schedules where portfolio_uuid in #{portfolioUuid, jdbcType=VARCHAR} + separator="," close=")">#{portfolioUuid, jdbcType=VARCHAR} - - delete from report_subscriptions where portfolio_uuid in #{portfolioUuid, jdbcType=VARCHAR} + + delete from report_subscriptions where portfolio_uuid in #{portfolioUuid, jdbcType=VARCHAR} + + + + delete from anticipated_transitions where project_uuid = #{projectUuid,jdbcType=VARCHAR} and created_at < #{createdAtBefore,jdbcType=BIGINT} diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java index f3c835f0619..473531372f5 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java @@ -19,9 +19,12 @@ */ package org.sonar.db.purge; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Optional; import org.junit.Test; +import org.sonar.api.impl.utils.TestSystem2; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; @@ -31,10 +34,10 @@ import static org.assertj.core.api.Assertions.assertThat; public class PurgeConfigurationTest { @Test public void should_delete_all_closed_issues() { - PurgeConfiguration conf = new PurgeConfiguration("root", "project", 0, Optional.empty(), System2.INSTANCE, emptySet()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", 0, Optional.empty(), System2.INSTANCE, emptySet(), 0); assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); - conf = new PurgeConfiguration("root", "project", -1, Optional.empty(), System2.INSTANCE, emptySet()); + conf = new PurgeConfiguration("root", "project", -1, Optional.empty(), System2.INSTANCE, emptySet(), 0); assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); } @@ -42,7 +45,7 @@ public class PurgeConfigurationTest { public void should_delete_only_old_closed_issues() { Date now = DateUtils.parseDate("2013-05-18"); - PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.empty(), System2.INSTANCE, emptySet()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.empty(), System2.INSTANCE, emptySet(), 0); Date toDate = conf.maxLiveDateOfClosedIssues(now); assertThat(toDate.getYear()).isEqualTo(113);// =2013 @@ -52,7 +55,7 @@ public class PurgeConfigurationTest { @Test public void should_have_empty_branch_purge_date() { - PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.of(10), System2.INSTANCE, emptySet()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.of(10), System2.INSTANCE, emptySet(), 0); assertThat(conf.maxLiveDateOfInactiveBranches()).isNotEmpty(); long tenDaysAgo = DateUtils.addDays(new Date(System2.INSTANCE.now()), -10).getTime(); assertThat(conf.maxLiveDateOfInactiveBranches().get().getTime()).isBetween(tenDaysAgo - 5000, tenDaysAgo + 5000); @@ -60,8 +63,22 @@ public class PurgeConfigurationTest { @Test public void should_calculate_branch_purge_date() { - PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.empty(), System2.INSTANCE, emptySet()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.empty(), System2.INSTANCE, emptySet(), 0); assertThat(conf.maxLiveDateOfInactiveBranches()).isEmpty(); } + @Test + public void should_delete_only_old_anticipated_transitions() { + int anticipatedTransitionMaxAge = 30; + TestSystem2 system2 = new TestSystem2(); + system2.setNow(Instant.now().toEpochMilli()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.empty(), system2, emptySet(), anticipatedTransitionMaxAge); + + Instant toDate = conf.maxLiveDateOfAnticipatedTransitions(); + + assertThat(toDate) + .isBeforeOrEqualTo(Instant.now().minus(30, ChronoUnit.DAYS)) + .isAfter(Instant.now().minus(31, ChronoUnit.DAYS)); + } + } diff --git a/sonar-core/src/main/java/org/sonar/core/config/PurgeConstants.java b/sonar-core/src/main/java/org/sonar/core/config/PurgeConstants.java index 9960d91ac70..3a847552d96 100644 --- a/sonar-core/src/main/java/org/sonar/core/config/PurgeConstants.java +++ b/sonar-core/src/main/java/org/sonar/core/config/PurgeConstants.java @@ -19,15 +19,21 @@ */ package org.sonar.core.config; -public interface PurgeConstants { +public class PurgeConstants { + + public static final String HOURS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_DAY = "sonar.dbcleaner.hoursBeforeKeepingOnlyOneSnapshotByDay"; + public static final String WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_WEEK = "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByWeek"; + public static final String WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_MONTH = "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByMonth"; + public static final String WEEKS_BEFORE_KEEPING_ONLY_ANALYSES_WITH_VERSION = "sonar.dbcleaner.weeksBeforeKeepingOnlyAnalysesWithVersion"; + public static final String WEEKS_BEFORE_DELETING_ALL_SNAPSHOTS = "sonar.dbcleaner.weeksBeforeDeletingAllSnapshots"; + public static final String DAYS_BEFORE_DELETING_CLOSED_ISSUES = "sonar.dbcleaner.daysBeforeDeletingClosedIssues"; + public static final String DAYS_BEFORE_DELETING_INACTIVE_BRANCHES_AND_PRS = "sonar.dbcleaner.daysBeforeDeletingInactiveBranchesAndPRs"; + public static final String BRANCHES_TO_KEEP_WHEN_INACTIVE = "sonar.dbcleaner.branchesToKeepWhenInactive"; + public static final String AUDIT_HOUSEKEEPING_FREQUENCY = "sonar.dbcleaner.auditHousekeeping"; + public static final String DAYS_BEFORE_DELETING_ANTICIPATED_TRANSITIONS = "sonar.dbcleaner.daysBeforeDeletingAnticipatedTransitions"; + + private PurgeConstants() { + //class cannot be instantiated + } - String HOURS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_DAY = "sonar.dbcleaner.hoursBeforeKeepingOnlyOneSnapshotByDay"; - String WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_WEEK = "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByWeek"; - String WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_MONTH = "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByMonth"; - String WEEKS_BEFORE_KEEPING_ONLY_ANALYSES_WITH_VERSION = "sonar.dbcleaner.weeksBeforeKeepingOnlyAnalysesWithVersion"; - String WEEKS_BEFORE_DELETING_ALL_SNAPSHOTS = "sonar.dbcleaner.weeksBeforeDeletingAllSnapshots"; - String DAYS_BEFORE_DELETING_CLOSED_ISSUES = "sonar.dbcleaner.daysBeforeDeletingClosedIssues"; - String DAYS_BEFORE_DELETING_INACTIVE_BRANCHES_AND_PRS = "sonar.dbcleaner.daysBeforeDeletingInactiveBranchesAndPRs"; - String BRANCHES_TO_KEEP_WHEN_INACTIVE = "sonar.dbcleaner.branchesToKeepWhenInactive"; - String AUDIT_HOUSEKEEPING_FREQUENCY = "sonar.dbcleaner.auditHousekeeping"; } diff --git a/sonar-core/src/main/java/org/sonar/core/config/PurgeProperties.java b/sonar-core/src/main/java/org/sonar/core/config/PurgeProperties.java index 9a4dd798611..35a730d0f57 100644 --- a/sonar-core/src/main/java/org/sonar/core/config/PurgeProperties.java +++ b/sonar-core/src/main/java/org/sonar/core/config/PurgeProperties.java @@ -103,6 +103,17 @@ public final class PurgeProperties { .category(CoreProperties.CATEGORY_HOUSEKEEPING) .subCategory(CoreProperties.SUBCATEGORY_GENERAL) .index(6) + .build(), + + PropertyDefinition.builder(PurgeConstants.DAYS_BEFORE_DELETING_ANTICIPATED_TRANSITIONS) + .defaultValue("30") + .name("Delete anticipated transitions after") + .description("Anticipated transitions that have not been applied for more than this number of days will be deleted.") + .type(PropertyType.INTEGER) + .onQualifiers(Qualifiers.PROJECT) + .category(CoreProperties.CATEGORY_HOUSEKEEPING) + .subCategory(CoreProperties.SUBCATEGORY_GENERAL) + .index(7) .build()); } } diff --git a/sonar-core/src/test/java/org/sonar/core/config/PurgePropertiesTest.java b/sonar-core/src/test/java/org/sonar/core/config/PurgePropertiesTest.java index 08f4d5fd7ea..2a70980b85e 100644 --- a/sonar-core/src/test/java/org/sonar/core/config/PurgePropertiesTest.java +++ b/sonar-core/src/test/java/org/sonar/core/config/PurgePropertiesTest.java @@ -27,6 +27,6 @@ public class PurgePropertiesTest { @Test public void shouldGetExtensions() { - assertThat(PurgeProperties.all()).hasSize(6); + assertThat(PurgeProperties.all()).hasSize(7); } }