import com.google.common.annotations.VisibleForTesting;
import java.util.Collection;
import java.util.Date;
+import java.util.Optional;
import javax.annotation.CheckForNull;
import org.apache.commons.lang.time.DateUtils;
import org.sonar.api.config.Configuration;
private final IdUuidPair rootProjectIdUuid;
private final String[] scopesWithoutHistoricalData;
private final int maxAgeInDaysOfClosedIssues;
+ private final Optional<Integer> maxAgeInDaysOfInactiveShortLivingBranches;
private final System2 system2;
private final Collection<String> disabledComponentUuids;
public PurgeConfiguration(IdUuidPair rootProjectId, String[] scopesWithoutHistoricalData, int maxAgeInDaysOfClosedIssues,
- System2 system2, Collection<String> disabledComponentUuids) {
+ Optional<Integer> maxAgeInDaysOfInactiveShortLivingBranches, System2 system2, Collection<String> disabledComponentUuids) {
this.rootProjectIdUuid = rootProjectId;
this.scopesWithoutHistoricalData = scopesWithoutHistoricalData;
this.maxAgeInDaysOfClosedIssues = maxAgeInDaysOfClosedIssues;
this.system2 = system2;
this.disabledComponentUuids = disabledComponentUuids;
+ this.maxAgeInDaysOfInactiveShortLivingBranches = maxAgeInDaysOfInactiveShortLivingBranches;
}
public static PurgeConfiguration newDefaultPurgeConfiguration(Configuration config, IdUuidPair idUuidPair, Collection<String> disabledComponentUuids) {
if (config.getBoolean(PurgeConstants.PROPERTY_CLEAN_DIRECTORY).orElse(false)) {
scopes = new String[] {Scopes.DIRECTORY, Scopes.FILE};
}
- return new PurgeConfiguration(idUuidPair, scopes, config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES).get(), System2.INSTANCE, disabledComponentUuids);
+ return new PurgeConfiguration(idUuidPair, scopes, config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES).get(),
+ config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_INACTIVE_SHORT_LIVING_BRANCHES), System2.INSTANCE, disabledComponentUuids);
}
public IdUuidPair rootProjectIdUuid() {
return maxLiveDateOfClosedIssues(new Date(system2.now()));
}
+ public Optional<Date> maxLiveDateOfInactiveShortLivingBranches() {
+ return maxAgeInDaysOfInactiveShortLivingBranches.map(age -> DateUtils.addDays(new Date(system2.now()), -age));
+ }
+
@VisibleForTesting
@CheckForNull
Date maxLiveDateOfClosedIssues(Date now) {
import java.util.Collections;
import java.util.Date;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import org.sonar.api.utils.System2;
import org.sonar.api.utils.log.Logger;
purgeAnalyses(commands, rootUuid);
purgeDisabledComponents(session, conf, listener);
deleteOldClosedIssues(conf, mapper, listener);
+ purgeStaleBranches(commands, conf, mapper, rootUuid);
+ }
+
+ private static void purgeStaleBranches(PurgeCommands commands, PurgeConfiguration conf, PurgeMapper mapper, String rootUuid) {
+ Optional<Date> maxDate = conf.maxLiveDateOfInactiveShortLivingBranches();
+ if (!maxDate.isPresent()) {
+ // not available if branch plugin is not installed
+ return;
+ }
+ LOG.debug("<- Purge stale branches");
+
+ List<String> branchUuids = mapper.selectStaleShortLivingBranches(rootUuid, dateToLong(maxDate.get()));
+
+ for (String branchUuid : branchUuids) {
+ deleteRootComponent(branchUuid, mapper, commands);
+ }
}
private static void purgeAnalyses(PurgeCommands commands, String rootUuid) {
List<String> selectOldClosedIssueKeys(@Param("projectUuid") String projectUuid, @Nullable @Param("toDate") Long toDate);
+ List<String> selectStaleShortLivingBranches(@Param("mainBranchProjectUuid") String mainBranchProjectUuid, @Param("toDate") Long toDate);
+
void deleteIssuesFromKeys(@Param("keys") List<String> keys);
void deleteIssueChangesFromIssueKeys(@Param("issueKeys") List<String> issueKeys);
and not exists(select e.id from events e where e.analysis_uuid=s.uuid)
</select>
+ <select id="selectStaleShortLivingBranches" parameterType="map" resultType="String">
+ select
+ pb.uuid
+ from
+ project_branches pb
+ where
+ pb.project_uuid=#{mainBranchProjectUuid,jdbcType=VARCHAR}
+ and pb.branch_type='SHORT'
+ and pb.updated_at < #{toDate}
+ </select>
+
<select id="selectMetricIdsWithoutHistoricalData" resultType="long">
select id from metrics where delete_historical_data=${_true}
</select>
- <select id="selectRootAndModulesOrSubviewsByProjectUuid" resultType="org.sonar.db.purge.IdUuidPair" parameterType="String">
+ <select id="selectRootAndModulesOrSubviewsByProjectUuid" resultType="IdUuidPair" parameterType="String">
select
p.id, p.uuid
from
import java.util.Arrays;
import java.util.function.Consumer;
import javax.annotation.Nullable;
-import org.sonar.core.util.Uuids;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;
import static com.google.common.base.Preconditions.checkState;
import static java.util.Arrays.asList;
-import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
-import static org.sonar.db.component.BranchKeyType.BRANCH;
import static org.sonar.db.component.BranchType.LONG;
import static org.sonar.db.component.ComponentTesting.newApplication;
+import static org.sonar.db.component.ComponentTesting.newBranchDto;
import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto;
import static org.sonar.db.component.ComponentTesting.newProjectBranch;
import static org.sonar.db.component.ComponentTesting.newPublicProjectDto;
@SafeVarargs
public final ComponentDto insertMainBranch(OrganizationDto organization, Consumer<ComponentDto>... dtoPopulators) {
ComponentDto project = newPrivateProjectDto(organization);
- BranchDto branchDto = new BranchDto()
- .setKey(null)
- .setUuid(project.uuid())
- .setProjectUuid(project.projectUuid())
- .setKeeType(BRANCH)
- .setBranchType(LONG);
+ BranchDto branchDto = newBranchDto(project, LONG);
Arrays.stream(dtoPopulators).forEach(dtoPopulator -> dtoPopulator.accept(project));
insertComponent(project);
dbClient.branchDao().insert(dbSession, branchDto);
@SafeVarargs
public final ComponentDto insertProjectBranch(ComponentDto project, Consumer<BranchDto>... dtoPopulators) {
- String uuid = Uuids.createFast();
- BranchDto branchDto = new BranchDto()
- .setKey("branch_" + randomAlphanumeric(248))
- .setUuid(uuid)
- // MainBranchProjectUuid will be null if it's a main branch
- .setProjectUuid(firstNonNull(project.getMainBranchProjectUuid(), project.projectUuid()))
- .setKeeType(BRANCH)
- .setBranchType(LONG);
+ // MainBranchProjectUuid will be null if it's a main branch
+ BranchDto branchDto = newBranchDto(firstNonNull(project.getMainBranchProjectUuid(), project.projectUuid()), LONG);
Arrays.stream(dtoPopulators).forEach(dtoPopulator -> dtoPopulator.accept(branchDto));
ComponentDto branch = newProjectBranch(project, branchDto);
insertComponent(branch);
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
+import static org.sonar.db.component.BranchKeyType.BRANCH;
import static org.sonar.db.component.ComponentDto.UUID_PATH_SEPARATOR;
public class ComponentTesting {
.setPrivate(moduleOrProject.isPrivate());
}
+ public static BranchDto newBranchDto(@Nullable String projectUuid, BranchType branchType) {
+ String key = projectUuid == null ? null : "branch_" + randomAlphanumeric(248);
+ return new BranchDto()
+ .setKey(key)
+ .setUuid(Uuids.createFast())
+ // MainBranchProjectUuid will be null if it's a main branch
+ .setProjectUuid(projectUuid)
+ .setKeeType(BRANCH)
+ .setBranchType(branchType);
+ }
+
+ public static BranchDto newBranchDto(ComponentDto branchComponent, BranchType branchType) {
+ boolean isMain = branchComponent.getMainBranchProjectUuid() == null;
+ String projectUuid = isMain ? branchComponent.uuid() : branchComponent.getMainBranchProjectUuid();
+ String key = isMain ? null : "branch_" + randomAlphanumeric(248);
+
+ return new BranchDto()
+ .setKey(key)
+ .setUuid(branchComponent.uuid())
+ // MainBranchProjectUuid will be null if it's a main branch
+ .setProjectUuid(projectUuid)
+ .setKeeType(BRANCH)
+ .setBranchType(branchType);
+ }
+
public static ComponentDto newProjectBranch(ComponentDto project, BranchDto branchDto) {
checkArgument(project.qualifier().equals(Qualifiers.PROJECT));
checkArgument(project.getMainBranchProjectUuid() == null);
import java.util.Collections;
import java.util.Date;
+import java.util.Optional;
import org.junit.Test;
import org.sonar.api.config.PropertyDefinitions;
import org.sonar.api.config.internal.MapSettings;
public class PurgeConfigurationTest {
@Test
public void should_delete_all_closed_issues() {
- PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 0, System2.INSTANCE, Collections.emptyList());
+ PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 0, Optional.empty(), System2.INSTANCE, Collections.emptyList());
assertThat(conf.maxLiveDateOfClosedIssues()).isNull();
- conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], -1, System2.INSTANCE, Collections.emptyList());
+ conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], -1, Optional.empty(), System2.INSTANCE, Collections.emptyList());
assertThat(conf.maxLiveDateOfClosedIssues()).isNull();
}
public void should_delete_only_old_closed_issues() {
Date now = DateUtils.parseDate("2013-05-18");
- PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30, System2.INSTANCE, Collections.emptyList());
+ PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30, Optional.empty(), System2.INSTANCE, Collections.emptyList());
Date toDate = conf.maxLiveDateOfClosedIssues(now);
assertThat(toDate.getYear()).isEqualTo(113);// =2013
assertThat(toDate.getDate()).isEqualTo(18);
}
+ @Test
+ public void should_have_empty_branch_purge_date() {
+ PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30, Optional.of(10), System2.INSTANCE, Collections.emptyList());
+ assertThat(conf.maxLiveDateOfInactiveShortLivingBranches()).isNotEmpty();
+ long tenDaysAgo = DateUtils.addDays(new Date(System2.INSTANCE.now()), -10).getTime();
+ assertThat(conf.maxLiveDateOfInactiveShortLivingBranches().get().getTime()).isBetween(tenDaysAgo - 5000, tenDaysAgo + 5000);
+ }
+
+ @Test
+ public void should_calculate_branch_purge_date() {
+ PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30, Optional.empty(), System2.INSTANCE, Collections.emptyList());
+ assertThat(conf.maxLiveDateOfInactiveShortLivingBranches()).isEmpty();
+ }
+
@Test
public void do_not_delete_directory_by_default() {
MapSettings settings = new MapSettings(new PropertyDefinitions(PurgeProperties.all()));
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Date;
import java.util.List;
+import java.util.Optional;
import java.util.Random;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.lang.time.DateUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeQueueDto.Status;
+import org.sonar.db.component.BranchType;
import org.sonar.db.component.ComponentDbTester;
import org.sonar.db.component.ComponentDto;
import org.sonar.db.component.ComponentTesting;
dbTester.assertDbUnit(getClass(), "shouldPurgeProject-result.xml", "projects", "snapshots");
}
+ @Test
+ public void should_purge_inactive_short_living_branches() {
+ when(system2.now()).thenReturn(new Date().getTime());
+ RuleDefinitionDto rule = dbTester.rules().insert();
+ ComponentDto project = dbTester.components().insertMainBranch();
+ ComponentDto longBranch = dbTester.components().insertProjectBranch(project);
+ ComponentDto recentShortBranch = dbTester.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.SHORT));
+
+ // short branch with other components and issues, updated 31 days ago
+ when(system2.now()).thenReturn(DateUtils.addDays(new Date(), -31).getTime());
+ ComponentDto shortBranch = dbTester.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.SHORT));
+ ComponentDto module = dbTester.components().insertComponent(newModuleDto(shortBranch));
+ ComponentDto subModule = dbTester.components().insertComponent(newModuleDto(module));
+ ComponentDto file = dbTester.components().insertComponent(newFileDto(subModule));
+ dbTester.issues().insert(rule, shortBranch, file);
+ dbTester.issues().insert(rule, shortBranch, subModule);
+ dbTester.issues().insert(rule, shortBranch, module);
+
+ // back to present
+ when(system2.now()).thenReturn(new Date().getTime());
+ underTest.purge(dbSession, newConfigurationWith30Days(system2, project.uuid()), PurgeListener.EMPTY, new PurgeProfiler());
+ dbSession.commit();
+
+ assertThat(getUuidsInTableProjects()).containsOnly(project.uuid(), longBranch.uuid(), recentShortBranch.uuid());
+ }
+
@Test
public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() {
dbTester.prepareDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml");
- PurgeConfiguration conf = new PurgeConfiguration(
- new IdUuidPair(THE_PROJECT_ID, "ABCD"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, 30, System2.INSTANCE, Collections.emptyList());
+ PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "ABCD"), new String[] {Scopes.DIRECTORY, Scopes.FILE},
+ 30, Optional.of(30), System2.INSTANCE, Collections.emptyList());
underTest.purge(dbSession, conf, PurgeListener.EMPTY, new PurgeProfiler());
dbSession.commit();
public void close_issues_clean_index_and_file_sources_of_disabled_components_specified_by_uuid_in_configuration() {
dbTester.prepareDbUnit(getClass(), "close_issues_clean_index_and_files_sources_of_specified_components.xml");
when(system2.now()).thenReturn(1450000000000L);
- underTest.purge(dbSession, newConfigurationWith30Days(system2, "P1", "EFGH", "GHIJ"), PurgeListener.EMPTY, new PurgeProfiler());
+ underTest.purge(dbSession, newConfigurationWith30Days(system2, THE_PROJECT_UUID, "P1", "EFGH", "GHIJ"), PurgeListener.EMPTY, new PurgeProfiler());
dbSession.commit();
dbTester.assertDbUnit(getClass(), "close_issues_clean_index_and_files_sources_of_specified_components-result.xml",
new String[] {"issue_close_date", "issue_update_date"},
@Test
public void should_delete_all_closed_issues() {
dbTester.prepareDbUnit(getClass(), "should_delete_all_closed_issues.xml");
- PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "1"), new String[0], 0, System2.INSTANCE, Collections.emptyList());
+ PurgeConfiguration conf = new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "1"), new String[0],
+ 0, Optional.empty(), System2.INSTANCE, Collections.emptyList());
underTest.purge(dbSession, conf, PurgeListener.EMPTY, new PurgeProfiler());
dbSession.commit();
dbTester.assertDbUnit(getClass(), "should_delete_all_closed_issues-result.xml", "issues", "issue_changes");
}
private static PurgeConfiguration newConfigurationWith30Days() {
- return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, THE_PROJECT_UUID), new String[0], 30, System2.INSTANCE, Collections.emptyList());
+ return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, THE_PROJECT_UUID), new String[0], 30, Optional.of(30), System2.INSTANCE, Collections.emptyList());
}
- private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String... disabledComponentUuids) {
- return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, THE_PROJECT_UUID), new String[0], 30, system2, asList(disabledComponentUuids));
+ private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String rootProjectUuid, String... disabledComponentUuids) {
+ return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, rootProjectUuid), new String[0], 30, Optional.of(30), system2, asList(disabledComponentUuids));
}
}
String WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_MONTH = "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByMonth";
String WEEKS_BEFORE_DELETING_ALL_SNAPSHOTS = "sonar.dbcleaner.weeksBeforeDeletingAllSnapshots";
String DAYS_BEFORE_DELETING_CLOSED_ISSUES = "sonar.dbcleaner.daysBeforeDeletingClosedIssues";
+ String DAYS_BEFORE_DELETING_INACTIVE_SHORT_LIVING_BRANCHES = "sonar.dbcleaner.daysBeforeDeletingInactiveShortLivingBranches";
}
projects / sonarqube.git / commitdiff