diff options
Diffstat (limited to 'server')
8 files changed, 128 insertions, 30 deletions
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java index 2d30880fc0a..c9d903f883f 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java @@ -22,6 +22,7 @@ package org.sonar.db.purge; import com.google.common.annotations.VisibleForTesting; import java.util.Collection; import java.util.Date; +import java.util.Optional; import javax.annotation.CheckForNull; import org.apache.commons.lang.time.DateUtils; import org.sonar.api.config.Configuration; @@ -34,16 +35,18 @@ public class PurgeConfiguration { private final IdUuidPair rootProjectIdUuid; private final String[] scopesWithoutHistoricalData; private final int maxAgeInDaysOfClosedIssues; + private final Optional<Integer> maxAgeInDaysOfInactiveShortLivingBranches; private final System2 system2; private final Collection<String> disabledComponentUuids; public PurgeConfiguration(IdUuidPair rootProjectId, String[] scopesWithoutHistoricalData, int maxAgeInDaysOfClosedIssues, - System2 system2, Collection<String> disabledComponentUuids) { + Optional<Integer> maxAgeInDaysOfInactiveShortLivingBranches, System2 system2, Collection<String> disabledComponentUuids) { this.rootProjectIdUuid = rootProjectId; this.scopesWithoutHistoricalData = scopesWithoutHistoricalData; this.maxAgeInDaysOfClosedIssues = maxAgeInDaysOfClosedIssues; this.system2 = system2; this.disabledComponentUuids = disabledComponentUuids; + this.maxAgeInDaysOfInactiveShortLivingBranches = maxAgeInDaysOfInactiveShortLivingBranches; } public static PurgeConfiguration newDefaultPurgeConfiguration(Configuration config, IdUuidPair idUuidPair, Collection<String> disabledComponentUuids) { @@ -51,7 +54,8 @@ public class PurgeConfiguration { if (config.getBoolean(PurgeConstants.PROPERTY_CLEAN_DIRECTORY).orElse(false)) { scopes = new String[] {Scopes.DIRECTORY, Scopes.FILE}; } - return new PurgeConfiguration(idUuidPair, scopes, config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES).get(), System2.INSTANCE, disabledComponentUuids); + return new PurgeConfiguration(idUuidPair, scopes, config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES).get(), + config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_INACTIVE_SHORT_LIVING_BRANCHES), System2.INSTANCE, disabledComponentUuids); } public IdUuidPair rootProjectIdUuid() { @@ -71,6 +75,10 @@ public class PurgeConfiguration { return maxLiveDateOfClosedIssues(new Date(system2.now())); } + public Optional<Date> maxLiveDateOfInactiveShortLivingBranches() { + return maxAgeInDaysOfInactiveShortLivingBranches.map(age -> DateUtils.addDays(new Date(system2.now()), -age)); + } + @VisibleForTesting @CheckForNull Date maxLiveDateOfClosedIssues(Date now) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java index 67fb3b701c6..4a9fc5d79b8 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Optional; import java.util.Set; import org.sonar.api.utils.System2; import org.sonar.api.utils.log.Logger; @@ -67,6 +68,22 @@ public class PurgeDao implements Dao { purgeAnalyses(commands, rootUuid); purgeDisabledComponents(session, conf, listener); deleteOldClosedIssues(conf, mapper, listener); + purgeStaleBranches(commands, conf, mapper, rootUuid); + } + + private static void purgeStaleBranches(PurgeCommands commands, PurgeConfiguration conf, PurgeMapper mapper, String rootUuid) { + Optional<Date> maxDate = conf.maxLiveDateOfInactiveShortLivingBranches(); + if (!maxDate.isPresent()) { + // not available if branch plugin is not installed + return; + } + LOG.debug("<- Purge stale branches"); + + List<String> branchUuids = mapper.selectStaleShortLivingBranches(rootUuid, dateToLong(maxDate.get())); + + for (String branchUuid : branchUuids) { + deleteRootComponent(branchUuid, mapper, commands); + } } private static void purgeAnalyses(PurgeCommands commands, String rootUuid) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java index 23d68ee1bc1..86928616be0 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -78,6 +78,8 @@ public interface PurgeMapper { List<String> selectOldClosedIssueKeys(@Param("projectUuid") String projectUuid, @Nullable @Param("toDate") Long toDate); + List<String> selectStaleShortLivingBranches(@Param("mainBranchProjectUuid") String mainBranchProjectUuid, @Param("toDate") Long toDate); + void deleteIssuesFromKeys(@Param("keys") List<String> keys); void deleteIssueChangesFromIssueKeys(@Param("issueKeys") List<String> issueKeys); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index d22fc0b973e..026672b7190 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -47,11 +47,22 @@ and not exists(select e.id from events e where e.analysis_uuid=s.uuid) </select> + <select id="selectStaleShortLivingBranches" parameterType="map" resultType="String"> + select + pb.uuid + from + project_branches pb + where + pb.project_uuid=#{mainBranchProjectUuid,jdbcType=VARCHAR} + and pb.branch_type='SHORT' + and pb.updated_at < #{toDate} + </select> + <select id="selectMetricIdsWithoutHistoricalData" resultType="long"> select id from metrics where delete_historical_data=${_true} </select> - <select id="selectRootAndModulesOrSubviewsByProjectUuid" resultType="org.sonar.db.purge.IdUuidPair" parameterType="String"> + <select id="selectRootAndModulesOrSubviewsByProjectUuid" resultType="IdUuidPair" parameterType="String"> select p.id, p.uuid from diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentDbTester.java b/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentDbTester.java index c06c714c74d..172cad17f47 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentDbTester.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentDbTester.java @@ -22,7 +22,6 @@ package org.sonar.db.component; import java.util.Arrays; import java.util.function.Consumer; import javax.annotation.Nullable; -import org.sonar.core.util.Uuids; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.DbTester; @@ -30,10 +29,9 @@ import org.sonar.db.organization.OrganizationDto; import static com.google.common.base.Preconditions.checkState; import static java.util.Arrays.asList; -import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; -import static org.sonar.db.component.BranchKeyType.BRANCH; import static org.sonar.db.component.BranchType.LONG; import static org.sonar.db.component.ComponentTesting.newApplication; +import static org.sonar.db.component.ComponentTesting.newBranchDto; import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; import static org.sonar.db.component.ComponentTesting.newProjectBranch; import static org.sonar.db.component.ComponentTesting.newPublicProjectDto; @@ -209,12 +207,7 @@ public class ComponentDbTester { @SafeVarargs public final ComponentDto insertMainBranch(OrganizationDto organization, Consumer<ComponentDto>... dtoPopulators) { ComponentDto project = newPrivateProjectDto(organization); - BranchDto branchDto = new BranchDto() - .setKey(null) - .setUuid(project.uuid()) - .setProjectUuid(project.projectUuid()) - .setKeeType(BRANCH) - .setBranchType(LONG); + BranchDto branchDto = newBranchDto(project, LONG); Arrays.stream(dtoPopulators).forEach(dtoPopulator -> dtoPopulator.accept(project)); insertComponent(project); dbClient.branchDao().insert(dbSession, branchDto); @@ -224,14 +217,8 @@ public class ComponentDbTester { @SafeVarargs public final ComponentDto insertProjectBranch(ComponentDto project, Consumer<BranchDto>... dtoPopulators) { - String uuid = Uuids.createFast(); - BranchDto branchDto = new BranchDto() - .setKey("branch_" + randomAlphanumeric(248)) - .setUuid(uuid) - // MainBranchProjectUuid will be null if it's a main branch - .setProjectUuid(firstNonNull(project.getMainBranchProjectUuid(), project.projectUuid())) - .setKeeType(BRANCH) - .setBranchType(LONG); + // MainBranchProjectUuid will be null if it's a main branch + BranchDto branchDto = newBranchDto(firstNonNull(project.getMainBranchProjectUuid(), project.projectUuid()), LONG); Arrays.stream(dtoPopulators).forEach(dtoPopulator -> dtoPopulator.accept(branchDto)); ComponentDto branch = newProjectBranch(project, branchDto); insertComponent(branch); diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentTesting.java b/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentTesting.java index 571b29cf47d..d57256a2a4d 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentTesting.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentTesting.java @@ -28,6 +28,8 @@ import org.sonar.db.organization.OrganizationDto; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; +import static org.sonar.db.component.BranchKeyType.BRANCH; import static org.sonar.db.component.ComponentDto.UUID_PATH_SEPARATOR; public class ComponentTesting { @@ -200,6 +202,31 @@ public class ComponentTesting { .setPrivate(moduleOrProject.isPrivate()); } + public static BranchDto newBranchDto(@Nullable String projectUuid, BranchType branchType) { + String key = projectUuid == null ? null : "branch_" + randomAlphanumeric(248); + return new BranchDto() + .setKey(key) + .setUuid(Uuids.createFast()) + // MainBranchProjectUuid will be null if it's a main branch + .setProjectUuid(projectUuid) + .setKeeType(BRANCH) + .setBranchType(branchType); + } + + public static BranchDto newBranchDto(ComponentDto branchComponent, BranchType branchType) { + boolean isMain = branchComponent.getMainBranchProjectUuid() == null; + String projectUuid = isMain ? branchComponent.uuid() : branchComponent.getMainBranchProjectUuid(); + String key = isMain ? null : "branch_" + randomAlphanumeric(248); + + return new BranchDto() + .setKey(key) + .setUuid(branchComponent.uuid()) + // MainBranchProjectUuid will be null if it's a main branch + .setProjectUuid(projectUuid) + .setKeeType(BRANCH) + .setBranchType(branchType); + } + public static ComponentDto newProjectBranch(ComponentDto project, BranchDto branchDto) { checkArgument(project.qualifier().equals(Qualifiers.PROJECT)); checkArgument(project.getMainBranchProjectUuid() == null); diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java index 57b89e8843e..a55713eb734 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java @@ -21,6 +21,7 @@ package org.sonar.db.purge; import java.util.Collections; import java.util.Date; +import java.util.Optional; import org.junit.Test; import org.sonar.api.config.PropertyDefinitions; import org.sonar.api.config.internal.MapSettings; @@ -35,10 +36,10 @@ import static org.assertj.core.api.Assertions.assertThat; public class PurgeConfigurationTest { @Test public void should_delete_all_closed_issues() { - PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 0, System2.INSTANCE, Collections.emptyList()); + PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 0, Optional.empty(), System2.INSTANCE, Collections.emptyList()); assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); - conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], -1, System2.INSTANCE, Collections.emptyList()); + conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], -1, Optional.empty(), System2.INSTANCE, Collections.emptyList()); assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); } @@ -46,7 +47,7 @@ public class PurgeConfigurationTest { public void should_delete_only_old_closed_issues() { Date now = DateUtils.parseDate("2013-05-18"); - PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30, System2.INSTANCE, Collections.emptyList()); + PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30, Optional.empty(), System2.INSTANCE, Collections.emptyList()); Date toDate = conf.maxLiveDateOfClosedIssues(now); assertThat(toDate.getYear()).isEqualTo(113);// =2013 @@ -55,6 +56,20 @@ public class PurgeConfigurationTest { } @Test + public void should_have_empty_branch_purge_date() { + PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30, Optional.of(10), System2.INSTANCE, Collections.emptyList()); + assertThat(conf.maxLiveDateOfInactiveShortLivingBranches()).isNotEmpty(); + long tenDaysAgo = DateUtils.addDays(new Date(System2.INSTANCE.now()), -10).getTime(); + assertThat(conf.maxLiveDateOfInactiveShortLivingBranches().get().getTime()).isBetween(tenDaysAgo - 5000, tenDaysAgo + 5000); + } + + @Test + public void should_calculate_branch_purge_date() { + PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30, Optional.empty(), System2.INSTANCE, Collections.emptyList()); + assertThat(conf.maxLiveDateOfInactiveShortLivingBranches()).isEmpty(); + } + + @Test public void do_not_delete_directory_by_default() { MapSettings settings = new MapSettings(new PropertyDefinitions(PurgeProperties.all())); settings.setProperty(PurgeConstants.PROPERTY_CLEAN_DIRECTORY, false); diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java index 903b59d161f..92259a36b2a 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java @@ -23,11 +23,14 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.List; +import java.util.Optional; import java.util.Random; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang.math.RandomUtils; +import org.apache.commons.lang.time.DateUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -41,6 +44,7 @@ import org.sonar.db.DbTester; import org.sonar.db.ce.CeActivityDto; import org.sonar.db.ce.CeQueueDto; import org.sonar.db.ce.CeQueueDto.Status; +import org.sonar.db.component.BranchType; import org.sonar.db.component.ComponentDbTester; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentTesting; @@ -100,10 +104,36 @@ public class PurgeDaoTest { } @Test + public void should_purge_inactive_short_living_branches() { + when(system2.now()).thenReturn(new Date().getTime()); + RuleDefinitionDto rule = dbTester.rules().insert(); + ComponentDto project = dbTester.components().insertMainBranch(); + ComponentDto longBranch = dbTester.components().insertProjectBranch(project); + ComponentDto recentShortBranch = dbTester.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.SHORT)); + + // short branch with other components and issues, updated 31 days ago + when(system2.now()).thenReturn(DateUtils.addDays(new Date(), -31).getTime()); + ComponentDto shortBranch = dbTester.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.SHORT)); + ComponentDto module = dbTester.components().insertComponent(newModuleDto(shortBranch)); + ComponentDto subModule = dbTester.components().insertComponent(newModuleDto(module)); + ComponentDto file = dbTester.components().insertComponent(newFileDto(subModule)); + dbTester.issues().insert(rule, shortBranch, file); + dbTester.issues().insert(rule, shortBranch, subModule); + dbTester.issues().insert(rule, shortBranch, module); + + // back to present + when(system2.now()).thenReturn(new Date().getTime()); + underTest.purge(dbSession, newConfigurationWith30Days(system2, project.uuid()), PurgeListener.EMPTY, new PurgeProfiler()); + dbSession.commit(); + + assertThat(getUuidsInTableProjects()).containsOnly(project.uuid(), longBranch.uuid(), recentShortBranch.uuid()); + } + + @Test public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() { dbTester.prepareDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml"); - PurgeConfiguration conf = new PurgeConfiguration( - new IdUuidPair(THE_PROJECT_ID, "ABCD"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, 30, System2.INSTANCE, Collections.emptyList()); + PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "ABCD"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, + 30, Optional.of(30), System2.INSTANCE, Collections.emptyList()); underTest.purge(dbSession, conf, PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); @@ -115,7 +145,7 @@ public class PurgeDaoTest { public void close_issues_clean_index_and_file_sources_of_disabled_components_specified_by_uuid_in_configuration() { dbTester.prepareDbUnit(getClass(), "close_issues_clean_index_and_files_sources_of_specified_components.xml"); when(system2.now()).thenReturn(1450000000000L); - underTest.purge(dbSession, newConfigurationWith30Days(system2, "P1", "EFGH", "GHIJ"), PurgeListener.EMPTY, new PurgeProfiler()); + underTest.purge(dbSession, newConfigurationWith30Days(system2, THE_PROJECT_UUID, "P1", "EFGH", "GHIJ"), PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); dbTester.assertDbUnit(getClass(), "close_issues_clean_index_and_files_sources_of_specified_components-result.xml", new String[] {"issue_close_date", "issue_update_date"}, @@ -313,7 +343,8 @@ public class PurgeDaoTest { @Test public void should_delete_all_closed_issues() { dbTester.prepareDbUnit(getClass(), "should_delete_all_closed_issues.xml"); - PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "1"), new String[0], 0, System2.INSTANCE, Collections.emptyList()); + PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "1"), new String[0], + 0, Optional.empty(), System2.INSTANCE, Collections.emptyList()); underTest.purge(dbSession, conf, PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); dbTester.assertDbUnit(getClass(), "should_delete_all_closed_issues-result.xml", "issues", "issue_changes"); @@ -582,11 +613,11 @@ public class PurgeDaoTest { } private static PurgeConfiguration newConfigurationWith30Days() { - return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, THE_PROJECT_UUID), new String[0], 30, System2.INSTANCE, Collections.emptyList()); + return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, THE_PROJECT_UUID), new String[0], 30, Optional.of(30), System2.INSTANCE, Collections.emptyList()); } - private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String... disabledComponentUuids) { - return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, THE_PROJECT_UUID), new String[0], 30, system2, asList(disabledComponentUuids)); + private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String rootProjectUuid, String... disabledComponentUuids) { + return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, rootProjectUuid), new String[0], 30, Optional.of(30), system2, asList(disabledComponentUuids)); } } |