From b64932e6d07560db391a4fe3c849529fd223eb75 Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Lesaint?= Date: Mon, 10 Jun 2019 13:58:23 +0200 Subject: [PATCH] SONAR-11745 deletion of disable component's child tables is resilient --- .../component/DisabledComponentsHolder.java | 4 +- .../DisabledComponentsHolderImpl.java | 8 +-- .../MutableDisabledComponentsHolder.java | 4 +- .../projectanalysis/purge/ProjectCleaner.java | 4 +- .../DisabledComponentsHolderImplTest.java | 8 +-- .../purge/ProjectCleanerTest.java | 8 +-- .../org/sonar/db/purge/PurgeCommands.java | 57 ++++++++++++++++++- .../sonar/db/purge/PurgeConfiguration.java | 9 +-- .../java/org/sonar/db/purge/PurgeDao.java | 37 +++++------- .../java/org/sonar/db/purge/PurgeMapper.java | 7 +++ .../org/sonar/db/purge/PurgeMapper.xml | 32 +++++++++++ .../org/sonar/db/purge/PurgeCommandsTest.java | 22 +++---- .../db/purge/PurgeConfigurationTest.java | 14 ++--- .../java/org/sonar/db/purge/PurgeDaoTest.java | 51 +++++++++++------ 14 files changed, 182 insertions(+), 83 deletions(-) diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolder.java index 95de5e505dc..08be8fc17e9 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolder.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolder.java @@ -19,10 +19,10 @@ */ package org.sonar.ce.task.projectanalysis.component; -import java.util.Collection; +import java.util.Set; public interface DisabledComponentsHolder { - Collection getUuids(); + Set getUuids(); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolderImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolderImpl.java index d9104470b0c..e697f0385f3 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolderImpl.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolderImpl.java @@ -19,22 +19,22 @@ */ package org.sonar.ce.task.projectanalysis.component; -import java.util.Collection; +import java.util.Set; import static com.google.common.base.Preconditions.checkState; public class DisabledComponentsHolderImpl implements MutableDisabledComponentsHolder { - private Collection uuids; + private Set uuids; @Override - public Collection getUuids() { + public Set getUuids() { checkState(uuids != null, "UUIDs have not been set in repository"); return uuids; } @Override - public void setUuids(Collection uuids) { + public void setUuids(Set uuids) { checkState(this.uuids == null, "UUIDs have already been set in repository"); this.uuids = uuids; } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/MutableDisabledComponentsHolder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/MutableDisabledComponentsHolder.java index 62c4aaa6f3a..a6de28d8946 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/MutableDisabledComponentsHolder.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/MutableDisabledComponentsHolder.java @@ -19,10 +19,10 @@ */ package org.sonar.ce.task.projectanalysis.component; -import java.util.Collection; +import java.util.Set; public interface MutableDisabledComponentsHolder extends DisabledComponentsHolder { - void setUuids(Collection uuids); + void setUuids(Set uuids); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/purge/ProjectCleaner.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/purge/ProjectCleaner.java index 6d462db18a0..15c246994b9 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/purge/ProjectCleaner.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/purge/ProjectCleaner.java @@ -19,7 +19,7 @@ */ package org.sonar.ce.task.projectanalysis.purge; -import java.util.Collection; +import java.util.Set; import org.sonar.api.CoreProperties; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.config.Configuration; @@ -53,7 +53,7 @@ public class ProjectCleaner { this.purgeListener = purgeListener; } - public ProjectCleaner purge(DbSession session, String rootUuid, String projectUuid, Configuration projectConfig, Collection disabledComponentUuids) { + public ProjectCleaner purge(DbSession session, String rootUuid, String projectUuid, Configuration projectConfig, Set disabledComponentUuids) { long start = System.currentTimeMillis(); profiler.reset(); diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolderImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolderImplTest.java index db35ca06a4b..747ee0a3d33 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolderImplTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/DisabledComponentsHolderImplTest.java @@ -19,11 +19,11 @@ */ package org.sonar.ce.task.projectanalysis.component; +import com.google.common.collect.ImmutableSet; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; public class DisabledComponentsHolderImplTest { @@ -35,18 +35,18 @@ public class DisabledComponentsHolderImplTest { @Test public void set_and_get_uuids() { - underTest.setUuids(asList("U1", "U2")); + underTest.setUuids(ImmutableSet.of("U1", "U2")); assertThat(underTest.getUuids()).containsExactly("U1", "U2"); } @Test public void setUuids_fails_if_called_twice() { - underTest.setUuids(asList("U1", "U2")); + underTest.setUuids(ImmutableSet.of("U1", "U2")); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("UUIDs have already been set in repository"); - underTest.setUuids(asList("U1", "U2")); + underTest.setUuids(ImmutableSet.of("U1", "U2")); } @Test diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/purge/ProjectCleanerTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/purge/ProjectCleanerTest.java index f5fd0dabdd0..1315ab2f7d6 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/purge/ProjectCleanerTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/purge/ProjectCleanerTest.java @@ -32,7 +32,7 @@ import org.sonar.db.purge.PurgeListener; import org.sonar.db.purge.PurgeProfiler; import org.sonar.db.purge.period.DefaultPeriodCleaner; -import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; @@ -57,7 +57,7 @@ public class ProjectCleanerTest { public void no_profiling_when_property_is_false() { settings.setProperty(CoreProperties.PROFILING_LOG_PROPERTY, false); - underTest.purge(mock(DbSession.class), "root", "project", settings.asConfig(), emptyList()); + underTest.purge(mock(DbSession.class), "root", "project", settings.asConfig(), emptySet()); verify(profiler, never()).dump(anyLong(), any()); } @@ -66,7 +66,7 @@ public class ProjectCleanerTest { public void profiling_when_property_is_true() { settings.setProperty(CoreProperties.PROFILING_LOG_PROPERTY, true); - underTest.purge(mock(DbSession.class), "root", "project", settings.asConfig(), emptyList()); + underTest.purge(mock(DbSession.class), "root", "project", settings.asConfig(), emptySet()); verify(profiler).dump(anyLong(), any()); } @@ -75,7 +75,7 @@ public class ProjectCleanerTest { public void call_period_cleaner_index_client_and_purge_dao() { settings.setProperty(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES, 5); - underTest.purge(mock(DbSession.class), "root", "project", settings.asConfig(), emptyList()); + underTest.purge(mock(DbSession.class), "root", "project", settings.asConfig(), emptySet()); verify(periodCleaner).clean(any(), any(), any()); verify(dao).purge(any(), any(), any(), any()); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java index 2f4f6a127ce..25b100556c0 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -22,11 +22,17 @@ package org.sonar.db.purge; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.sonar.api.utils.System2; import org.sonar.db.DbSession; +import static org.sonar.db.DatabaseUtils.executeLargeInputs; + class PurgeCommands { private static final int MAX_SNAPSHOTS_PER_QUERY = 1000; @@ -35,16 +41,18 @@ class PurgeCommands { private final DbSession session; private final PurgeMapper purgeMapper; private final PurgeProfiler profiler; + private final System2 system2; - PurgeCommands(DbSession session, PurgeMapper purgeMapper, PurgeProfiler profiler) { + PurgeCommands(DbSession session, PurgeMapper purgeMapper, PurgeProfiler profiler, System2 system2) { this.session = session; this.purgeMapper = purgeMapper; this.profiler = profiler; + this.system2 = system2; } @VisibleForTesting - PurgeCommands(DbSession session, PurgeProfiler profiler) { - this(session, session.getMapper(PurgeMapper.class), profiler); + PurgeCommands(DbSession session, PurgeProfiler profiler, System2 system2) { + this(session, session.getMapper(PurgeMapper.class), profiler, system2); } List selectSnapshotUuids(PurgeSnapshotQuery query) { @@ -147,6 +155,49 @@ class PurgeCommands { profiler.stop(); } + void purgeDisabledComponents(String rootComponentUuid, Collection disabledComponentUuids, PurgeListener listener) { + Set missedDisabledComponentUuids = new HashSet<>(); + + profiler.start("purgeDisabledComponents (file_sources)"); + missedDisabledComponentUuids.addAll( + executeLargeInputs( + purgeMapper.selectDisabledComponentsWithFileSource(rootComponentUuid), + input -> { + purgeMapper.deleteFileSourcesByFileUuid(input); + return input; + })); + profiler.stop(); + + profiler.start("purgeDisabledComponents (unresolved_issues)"); + missedDisabledComponentUuids.addAll( + executeLargeInputs( + purgeMapper.selectDisabledComponentsWithUnresolvedIssues(rootComponentUuid), + input -> { + purgeMapper.resolveComponentIssuesNotAlreadyResolved(input, system2.now()); + return input; + })); + profiler.stop(); + + profiler.start("purgeDisabledComponents (live_measures)"); + missedDisabledComponentUuids.addAll( + executeLargeInputs( + purgeMapper.selectDisabledComponentsWithLiveMeasures(rootComponentUuid), + input -> { + purgeMapper.deleteLiveMeasuresByComponentUuids(input); + return input; + })); + profiler.stop(); + + session.commit(); + + // notify listener for any disabled component we found child data for which isn't part of the disabled components + // provided + missedDisabledComponentUuids.removeAll(disabledComponentUuids); + if (!missedDisabledComponentUuids.isEmpty()) { + listener.onComponentsDisabling(rootComponentUuid, missedDisabledComponentUuids); + } + } + private void deleteAnalysisDuplications(List> snapshotUuidsPartitions) { profiler.start("deleteAnalysisDuplications (duplications_index)"); snapshotUuidsPartitions.forEach(purgeMapper::deleteAnalysisDuplications); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java index 0f4d1b3b1cb..461aa2b3153 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.Optional; +import java.util.Set; import javax.annotation.CheckForNull; import org.sonar.api.config.Configuration; import org.sonar.api.resources.Scopes; @@ -39,10 +40,10 @@ public class PurgeConfiguration { private final int maxAgeInDaysOfClosedIssues; private final Optional maxAgeInDaysOfInactiveShortLivingBranches; private final System2 system2; - private final Collection disabledComponentUuids; + private final Set disabledComponentUuids; public PurgeConfiguration(String rootUuid, String projectUuid, Collection scopesWithoutHistoricalData, int maxAgeInDaysOfClosedIssues, - Optional maxAgeInDaysOfInactiveShortLivingBranches, System2 system2, Collection disabledComponentUuids) { + Optional maxAgeInDaysOfInactiveShortLivingBranches, System2 system2, Set disabledComponentUuids) { this.rootUuid = rootUuid; this.projectUuid = projectUuid; this.scopesWithoutHistoricalData = scopesWithoutHistoricalData; @@ -52,7 +53,7 @@ public class PurgeConfiguration { this.maxAgeInDaysOfInactiveShortLivingBranches = maxAgeInDaysOfInactiveShortLivingBranches; } - public static PurgeConfiguration newDefaultPurgeConfiguration(Configuration config, String rootUuid, String projectUuid, Collection disabledComponentUuids) { + public static PurgeConfiguration newDefaultPurgeConfiguration(Configuration config, String rootUuid, String projectUuid, Set disabledComponentUuids) { return new PurgeConfiguration(rootUuid, projectUuid, Arrays.asList(Scopes.DIRECTORY, Scopes.FILE), config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES).get(), config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_INACTIVE_SHORT_LIVING_BRANCHES), System2.INSTANCE, disabledComponentUuids); } @@ -77,7 +78,7 @@ public class PurgeConfiguration { return scopesWithoutHistoricalData; } - public Collection getDisabledComponentUuids() { + public Set getDisabledComponentUuids() { return disabledComponentUuids; } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java index 1f0c55ff2b0..915c56349e3 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -63,12 +63,12 @@ public class PurgeDao implements Dao { public void purge(DbSession session, PurgeConfiguration conf, PurgeListener listener, PurgeProfiler profiler) { PurgeMapper mapper = session.getMapper(PurgeMapper.class); - PurgeCommands commands = new PurgeCommands(session, mapper, profiler); + PurgeCommands commands = new PurgeCommands(session, mapper, profiler, system2); String rootUuid = conf.rootUuid(); deleteAbortedAnalyses(rootUuid, commands); deleteDataOfComponentsWithoutHistoricalData(session, rootUuid, conf.getScopesWithoutHistoricalData(), commands); purgeAnalyses(commands, rootUuid); - purgeDisabledComponents(session, mapper, conf, listener); + purgeDisabledComponents(commands, conf, listener); deleteOldClosedIssues(conf, mapper, listener); purgeOldCeActivities(rootUuid, commands); purgeOldCeScannerContexts(rootUuid, commands); @@ -102,6 +102,12 @@ public class PurgeDao implements Dao { commands.purgeAnalyses(analysisUuids); } + private static void purgeDisabledComponents(PurgeCommands commands, PurgeConfiguration conf, PurgeListener listener) { + String rootUuid = conf.rootUuid(); + listener.onComponentsDisabling(rootUuid, conf.getDisabledComponentUuids()); + commands.purgeDisabledComponents(rootUuid, conf.getDisabledComponentUuids(), listener); + } + private static void deleteOldClosedIssues(PurgeConfiguration conf, PurgeMapper mapper, PurgeListener listener) { Date toDate = conf.maxLiveDateOfClosedIssues(); String rootUuid = conf.rootUuid(); @@ -148,20 +154,6 @@ public class PurgeDao implements Dao { purgeCommands.deleteComponentMeasures(analysisUuids, componentWithoutHistoricalDataUuids); } - private void purgeDisabledComponents(DbSession session, PurgeMapper mapper, PurgeConfiguration conf, PurgeListener listener) { - executeLargeInputs(conf.getDisabledComponentUuids(), - input -> { - mapper.deleteFileSourcesByFileUuid(input); - mapper.resolveComponentIssuesNotAlreadyResolved(input, system2.now()); - mapper.deleteLiveMeasuresByComponentUuids(input); - return emptyList(); - }); - - listener.onComponentsDisabling(conf.rootUuid(), conf.getDisabledComponentUuids()); - - session.commit(); - } - private static void deleteOldDisabledComponents(PurgeCommands commands, PurgeMapper mapper, String rootUuid) { List disabledComponentsWithoutIssue = mapper.selectDisabledComponentsWithoutIssues(rootUuid); commands.deleteDisabledComponentsWithoutIssues(disabledComponentsWithoutIssue); @@ -180,7 +172,7 @@ public class PurgeDao implements Dao { public void purgeCeActivities(DbSession session, PurgeProfiler profiler) { PurgeMapper mapper = session.getMapper(PurgeMapper.class); - PurgeCommands commands = new PurgeCommands(session, mapper, profiler); + PurgeCommands commands = new PurgeCommands(session, mapper, profiler, system2); purgeOldCeActivities(null, commands); } @@ -191,7 +183,7 @@ public class PurgeDao implements Dao { public void purgeCeScannerContexts(DbSession session, PurgeProfiler profiler) { PurgeMapper mapper = session.getMapper(PurgeMapper.class); - PurgeCommands commands = new PurgeCommands(session, mapper, profiler); + PurgeCommands commands = new PurgeCommands(session, mapper, profiler, system2); purgeOldCeScannerContexts(null, commands); } @@ -200,7 +192,6 @@ public class PurgeDao implements Dao { commands.deleteCeScannerContextBefore(rootUuid, fourWeeksAgo.getTime()); } - private static final class ManualBaselineAnalysisFilter implements Predicate { private static final String[] NO_BASELINE = {null}; @@ -228,14 +219,14 @@ public class PurgeDao implements Dao { public void deleteBranch(DbSession session, String uuid) { PurgeProfiler profiler = new PurgeProfiler(); PurgeMapper purgeMapper = mapper(session); - PurgeCommands purgeCommands = new PurgeCommands(session, profiler); + PurgeCommands purgeCommands = new PurgeCommands(session, profiler, system2); deleteRootComponent(uuid, purgeMapper, purgeCommands); } public void deleteProject(DbSession session, String uuid) { PurgeProfiler profiler = new PurgeProfiler(); PurgeMapper purgeMapper = mapper(session); - PurgeCommands purgeCommands = new PurgeCommands(session, profiler); + PurgeCommands purgeCommands = new PurgeCommands(session, profiler, system2); session.getMapper(BranchMapper.class).selectByProjectUuid(uuid) .stream() @@ -283,7 +274,7 @@ public class PurgeDao implements Dao { } PurgeProfiler profiler = new PurgeProfiler(); - PurgeCommands purgeCommands = new PurgeCommands(dbSession, profiler); + PurgeCommands purgeCommands = new PurgeCommands(dbSession, profiler, system2); deleteNonRootComponentsInView(nonRootComponents, purgeCommands); } @@ -311,7 +302,7 @@ public class PurgeDao implements Dao { } public void deleteAnalyses(DbSession session, PurgeProfiler profiler, List analysisIdUuids) { - new PurgeCommands(session, profiler).deleteAnalyses(analysisIdUuids); + new PurgeCommands(session, profiler, system2).deleteAnalyses(analysisIdUuids); } private static PurgeMapper mapper(DbSession session) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java index 204794011d7..28ab1d58331 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -20,6 +20,7 @@ package org.sonar.db.purge; import java.util.List; +import java.util.Set; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.ibatis.annotations.Param; @@ -33,6 +34,12 @@ public interface PurgeMapper { */ List selectRootAndModulesOrSubviewsByProjectUuid(@Param("rootUuid") String rootUuid); + Set selectDisabledComponentsWithFileSource(@Param("projectUuid") String projectUuid); + + Set selectDisabledComponentsWithUnresolvedIssues(@Param("projectUuid") String projectUuid); + + Set selectDisabledComponentsWithLiveMeasures(@Param("projectUuid") String projectUuid); + void deleteAnalyses(@Param("analysisUuids") List analysisUuids); void deleteAnalysisProperties(@Param("analysisUuids") List analysisUuids); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index 9c0a301a450..3c521ac55e8 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -83,6 +83,38 @@ ) + + + + + + delete from project_measures where diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeCommandsTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeCommandsTest.java index dd0f8bedf95..849dd53dfe7 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeCommandsTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeCommandsTest.java @@ -24,6 +24,7 @@ import org.junit.After; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.System2; +import org.sonar.api.utils.internal.AlwaysIncreasingSystem2; import org.sonar.db.DbTester; import org.sonar.db.component.ComponentDto; import org.sonar.db.organization.OrganizationDto; @@ -41,6 +42,7 @@ public class PurgeCommandsTest { @Rule public DbTester dbTester = DbTester.create(System2.INSTANCE); + private AlwaysIncreasingSystem2 system2 = new AlwaysIncreasingSystem2(); private PurgeProfiler profiler = new PurgeProfiler(); /** @@ -56,7 +58,7 @@ public class PurgeCommandsTest { */ @Test public void should_not_fail_when_deleting_huge_number_of_analyses() { - new PurgeCommands(dbTester.getSession(), profiler).deleteAnalyses(getHugeNumberOfIdUuidPairs()); + new PurgeCommands(dbTester.getSession(), profiler, system2).deleteAnalyses(getHugeNumberOfIdUuidPairs()); // The goal of this test is only to check that the query do no fail, not to check result } @@ -67,7 +69,7 @@ public class PurgeCommandsTest { public void shouldPurgeAnalysis() { dbTester.prepareDbUnit(getClass(), "shouldPurgeAnalysis.xml"); - new PurgeCommands(dbTester.getSession(), profiler).purgeAnalyses(singletonList(new IdUuidPair(1, "u1"))); + new PurgeCommands(dbTester.getSession(), profiler, system2).purgeAnalyses(singletonList(new IdUuidPair(1, "u1"))); dbTester.assertDbUnit(getClass(), "shouldPurgeAnalysis-result.xml", "snapshots", "analysis_properties", "project_measures", "duplications_index", "events"); } @@ -76,7 +78,7 @@ public class PurgeCommandsTest { public void delete_wasted_measures_when_purging_analysis() { dbTester.prepareDbUnit(getClass(), "shouldDeleteWastedMeasuresWhenPurgingAnalysis.xml"); - new PurgeCommands(dbTester.getSession(), profiler).purgeAnalyses(singletonList(new IdUuidPair(1, "u1"))); + new PurgeCommands(dbTester.getSession(), profiler, system2).purgeAnalyses(singletonList(new IdUuidPair(1, "u1"))); dbTester.assertDbUnit(getClass(), "shouldDeleteWastedMeasuresWhenPurgingAnalysis-result.xml", "project_measures"); } @@ -86,7 +88,7 @@ public class PurgeCommandsTest { */ @Test public void should_not_fail_when_purging_huge_number_of_analyses() { - new PurgeCommands(dbTester.getSession(), profiler).purgeAnalyses(getHugeNumberOfIdUuidPairs()); + new PurgeCommands(dbTester.getSession(), profiler, system2).purgeAnalyses(getHugeNumberOfIdUuidPairs()); // The goal of this test is only to check that the query do no fail, not to check result } @@ -94,7 +96,7 @@ public class PurgeCommandsTest { public void shouldDeleteComponentsAndChildrenTables() { dbTester.prepareDbUnit(getClass(), "shouldDeleteResource.xml"); - PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler); + PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler, system2); purgeCommands.deleteComponents("uuid_1"); assertThat(dbTester.countRowsOfTable("projects")).isZero(); @@ -109,7 +111,7 @@ public class PurgeCommandsTest { public void shouldDeleteAnalyses() { dbTester.prepareDbUnit(getClass(), "shouldDeleteResource.xml"); - PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler); + PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler, system2); purgeCommands.deleteAnalyses("uuid_1"); assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(1); @@ -125,7 +127,7 @@ public class PurgeCommandsTest { public void shouldDeleteIssuesAndIssueChanges() { dbTester.prepareDbUnit(getClass(), "shouldDeleteResource.xml"); - PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler); + PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler, system2); purgeCommands.deleteIssues("uuid_1"); assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(1); @@ -141,7 +143,7 @@ public class PurgeCommandsTest { ComponentDto project = dbTester.components().insertPublicProject(organization); addPermissions(organization, project); - PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler); + PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler, system2); purgeCommands.deletePermissions(project.getId()); assertThat(dbTester.countRowsOfTable("group_roles")).isEqualTo(2); @@ -154,7 +156,7 @@ public class PurgeCommandsTest { ComponentDto project = dbTester.components().insertPrivateProject(organization); addPermissions(organization, project); - PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler); + PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler, system2); purgeCommands.deletePermissions(project.getId()); assertThat(dbTester.countRowsOfTable("group_roles")).isEqualTo(1); @@ -167,7 +169,7 @@ public class PurgeCommandsTest { ComponentDto project = dbTester.components().insertPublicPortfolio(organization); addPermissions(organization, project); - PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler); + PurgeCommands purgeCommands = new PurgeCommands(dbTester.getSession(), profiler, system2); purgeCommands.deletePermissions(project.getId()); assertThat(dbTester.countRowsOfTable("group_roles")).isEqualTo(2); diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java index 03b6751da88..4d83aa022e7 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java @@ -30,16 +30,16 @@ import org.sonar.api.utils.System2; import org.sonar.core.config.PurgeConstants; import org.sonar.core.config.PurgeProperties; -import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; import static org.assertj.core.api.Assertions.assertThat; public class PurgeConfigurationTest { @Test public void should_delete_all_closed_issues() { - PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptyList(), 0, Optional.empty(), System2.INSTANCE, emptyList()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptySet(), 0, Optional.empty(), System2.INSTANCE, emptySet()); assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); - conf = new PurgeConfiguration("root", "project", emptyList(), -1, Optional.empty(), System2.INSTANCE, emptyList()); + conf = new PurgeConfiguration("root", "project", emptySet(), -1, Optional.empty(), System2.INSTANCE, emptySet()); assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); } @@ -47,7 +47,7 @@ public class PurgeConfigurationTest { public void should_delete_only_old_closed_issues() { Date now = DateUtils.parseDate("2013-05-18"); - PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptyList(), 30, Optional.empty(), System2.INSTANCE, emptyList()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptySet(), 30, Optional.empty(), System2.INSTANCE, emptySet()); Date toDate = conf.maxLiveDateOfClosedIssues(now); assertThat(toDate.getYear()).isEqualTo(113);// =2013 @@ -57,7 +57,7 @@ public class PurgeConfigurationTest { @Test public void should_have_empty_branch_purge_date() { - PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptyList(), 30, Optional.of(10), System2.INSTANCE, emptyList()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptySet(), 30, Optional.of(10), System2.INSTANCE, emptySet()); assertThat(conf.maxLiveDateOfInactiveShortLivingBranches()).isNotEmpty(); long tenDaysAgo = DateUtils.addDays(new Date(System2.INSTANCE.now()), -10).getTime(); assertThat(conf.maxLiveDateOfInactiveShortLivingBranches().get().getTime()).isBetween(tenDaysAgo - 5000, tenDaysAgo + 5000); @@ -65,7 +65,7 @@ public class PurgeConfigurationTest { @Test public void should_calculate_branch_purge_date() { - PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptyList(), 30, Optional.empty(), System2.INSTANCE, emptyList()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptySet(), 30, Optional.empty(), System2.INSTANCE, emptySet()); assertThat(conf.maxLiveDateOfInactiveShortLivingBranches()).isEmpty(); } @@ -75,7 +75,7 @@ public class PurgeConfigurationTest { settings.setProperty(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES, 5); Date now = new Date(); - PurgeConfiguration underTest = PurgeConfiguration.newDefaultPurgeConfiguration(settings.asConfig(), "root", "project", emptyList()); + PurgeConfiguration underTest = PurgeConfiguration.newDefaultPurgeConfiguration(settings.asConfig(), "root", "project", emptySet()); assertThat(underTest.getScopesWithoutHistoricalData()) .containsExactlyInAnyOrder(Scopes.DIRECTORY, Scopes.FILE); diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java index 91724abf546..c9f4b619030 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java @@ -30,6 +30,7 @@ import java.util.Date; import java.util.List; import java.util.Optional; import java.util.Random; +import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -81,10 +82,12 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static java.time.ZoneOffset.UTC; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import static org.sonar.db.ce.CeTaskTypes.REPORT; @@ -212,7 +215,7 @@ public class PurgeDaoTest { public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() { db.prepareDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml"); PurgeConfiguration conf = new PurgeConfiguration("PROJECT_UUID", "PROJECT_UUID", asList(Scopes.DIRECTORY, Scopes.FILE), - 30, Optional.of(30), System2.INSTANCE, Collections.emptyList()); + 30, Optional.of(30), System2.INSTANCE, emptySet()); underTest.purge(dbSession, conf, PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); @@ -232,13 +235,13 @@ public class PurgeDaoTest { ComponentDto dir = db.components().insertComponent(newDirectory(module, "sub").setEnabled(false)); ComponentDto srcFile = db.components().insertComponent(newFileDto(module, dir).setEnabled(false)); ComponentDto testFile = db.components().insertComponent(newFileDto(module, dir).setEnabled(false)); - ComponentDto nonSelectedFile = db.components().insertComponent(newFileDto(module, dir).setEnabled(false)); + ComponentDto enabledFile = db.components().insertComponent(newFileDto(module, dir).setEnabled(true)); IssueDto openOnFile = db.issues().insert(rule, project, srcFile, issue -> issue.setStatus("OPEN")); IssueDto confirmOnFile = db.issues().insert(rule, project, srcFile, issue -> issue.setStatus("CONFIRM")); IssueDto openOnDir = db.issues().insert(rule, project, dir, issue -> issue.setStatus("OPEN")); IssueDto confirmOnDir = db.issues().insert(rule, project, dir, issue -> issue.setStatus("CONFIRM")); - IssueDto openOnNonSelected = db.issues().insert(rule, project, nonSelectedFile, issue -> issue.setStatus("OPEN")); - IssueDto confirmOnNonSelected = db.issues().insert(rule, project, nonSelectedFile, issue -> issue.setStatus("CONFIRM")); + IssueDto openOnEnabledComponent = db.issues().insert(rule, project, enabledFile, issue -> issue.setStatus("OPEN")); + IssueDto confirmOnEnabledComponent = db.issues().insert(rule, project, enabledFile, issue -> issue.setStatus("CONFIRM")); assertThat(db.countSql("select count(*) from snapshots where purge_status = 1")).isEqualTo(0); @@ -246,7 +249,7 @@ public class PurgeDaoTest { assertThat(db.countSql("select count(*) from issues where resolution = 'REMOVED'")).isEqualTo(0); db.fileSources().insertFileSource(srcFile); - FileSourceDto nonSelectedFileSource = db.fileSources().insertFileSource(nonSelectedFile); + FileSourceDto nonSelectedFileSource = db.fileSources().insertFileSource(enabledFile); assertThat(db.countRowsOfTable("file_sources")).isEqualTo(2); MetricDto metric1 = db.measures().insertMetric(); @@ -257,15 +260,20 @@ public class PurgeDaoTest { LiveMeasureDto liveMeasureMetric2OnDir = db.measures().insertLiveMeasure(dir, metric2); LiveMeasureDto liveMeasureMetric1OnProject = db.measures().insertLiveMeasure(project, metric1); LiveMeasureDto liveMeasureMetric2OnProject = db.measures().insertLiveMeasure(project, metric2); - LiveMeasureDto liveMeasureMetric1OnNonSelected = db.measures().insertLiveMeasure(nonSelectedFile, metric1); - LiveMeasureDto liveMeasureMetric2OnNonSelected = db.measures().insertLiveMeasure(nonSelectedFile, metric2); + LiveMeasureDto liveMeasureMetric1OnNonSelected = db.measures().insertLiveMeasure(enabledFile, metric1); + LiveMeasureDto liveMeasureMetric2OnNonSelected = db.measures().insertLiveMeasure(enabledFile, metric2); assertThat(db.countRowsOfTable("live_measures")).isEqualTo(8); + PurgeListener purgeListener = mock(PurgeListener.class); // back to present - underTest.purge(dbSession, newConfigurationWith30Days(system2, project.uuid(), project.uuid(), module.uuid(), dir.uuid(), srcFile.uuid(), testFile.uuid()), PurgeListener.EMPTY, - new PurgeProfiler()); + Set selectedComponentUuids = ImmutableSet.of(module.uuid(), srcFile.uuid(), testFile.uuid()); + underTest.purge(dbSession, newConfigurationWith30Days(system2, project.uuid(), project.uuid(), selectedComponentUuids), + purgeListener, new PurgeProfiler()); dbSession.commit(); + verify(purgeListener).onComponentsDisabling(project.uuid(), selectedComponentUuids); + verify(purgeListener).onComponentsDisabling(project.uuid(), ImmutableSet.of(dir.uuid())); + // set purge_status=1 for non-last snapshot assertThat(db.countSql("select count(*) from snapshots where purge_status = 1")).isEqualTo(1); @@ -276,7 +284,7 @@ public class PurgeDaoTest { .extracting(IssueDto::getStatus, IssueDto::getResolution) .containsExactlyInAnyOrder("CLOSED", "REMOVED"); } - for (IssueDto issue : Arrays.asList(openOnNonSelected, confirmOnNonSelected)) { + for (IssueDto issue : Arrays.asList(openOnEnabledComponent, confirmOnEnabledComponent)) { assertThat(db.getDbClient().issueDao().selectByKey(dbSession, issue.getKey()).get()) .extracting("status", "resolution") .containsExactlyInAnyOrder(issue.getStatus(), null); @@ -289,11 +297,11 @@ public class PurgeDaoTest { // deletes live measure of selected assertThat(db.countRowsOfTable("live_measures")).isEqualTo(4); List liveMeasureDtos = db.getDbClient().liveMeasureDao() - .selectByComponentUuidsAndMetricIds(dbSession, ImmutableSet.of(srcFile.uuid(), dir.uuid(), project.uuid(), nonSelectedFile.uuid()), + .selectByComponentUuidsAndMetricIds(dbSession, ImmutableSet.of(srcFile.uuid(), dir.uuid(), project.uuid(), enabledFile.uuid()), ImmutableSet.of(metric1.getId(), metric2.getId())); assertThat(liveMeasureDtos) .extracting(LiveMeasureDto::getComponentUuid) - .containsOnly(nonSelectedFile.uuid(), project.uuid()); + .containsOnly(enabledFile.uuid(), project.uuid()); assertThat(liveMeasureDtos) .extracting(LiveMeasureDto::getMetricId) .containsOnly(metric1.getId(), metric2.getId()); @@ -1019,15 +1027,18 @@ public class PurgeDaoTest { issue.setResolution(Issue.RESOLUTION_FIXED); issue.setIssueCloseDate(new Date()); }); + PurgeListener purgeListener = mock(PurgeListener.class); + Set disabledComponentUuids = ImmutableSet.of(disabledFileWithIssues.uuid(), disabledFileWithoutIssues.uuid()); underTest.purge(dbSession, - newConfigurationWith30Days(System2.INSTANCE, project.uuid(), disabledFileWithIssues.uuid(), disabledFileWithoutIssues.uuid()), - PurgeListener.EMPTY, new PurgeProfiler()); + newConfigurationWith30Days(System2.INSTANCE, project.uuid(), project.uuid(), disabledComponentUuids), + purgeListener, new PurgeProfiler()); assertThat(db.getDbClient().componentDao().selectByProjectUuid(project.uuid(), dbSession)) .extracting("uuid") .containsOnly(project.uuid(), enabledFileWithIssues.uuid(), disabledFileWithIssues.uuid(), enabledFileWithoutIssues.uuid()); + verify(purgeListener).onComponentsDisabling(project.uuid(), disabledComponentUuids); } @Test @@ -1072,7 +1083,7 @@ public class PurgeDaoTest { assertThat(selectActivity("NOT_OLD_ENOUGH_1")).isNotEmpty(); assertThat(selectTaskInput("NOT_OLD_ENOUGH_1")).isNotEmpty(); assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_1")).hasSize(1); - assertThat(scannerContextExists("NOT_OLD_ENOUGH_1")).isFalse(); // because more than 4 weeks old + assertThat(scannerContextExists("NOT_OLD_ENOUGH_1")).isFalse(); // because more than 4 weeks old assertThat(selectActivity("NOT_OLD_ENOUGH_2")).isNotEmpty(); assertThat(selectTaskInput("NOT_OLD_ENOUGH_2")).isNotEmpty(); assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_2")).hasSize(1); @@ -1659,11 +1670,15 @@ public class PurgeDaoTest { } private static PurgeConfiguration newConfigurationWith30Days() { - return new PurgeConfiguration(PROJECT_UUID, PROJECT_UUID, emptyList(), 30, Optional.of(30), System2.INSTANCE, Collections.emptyList()); + return new PurgeConfiguration(PROJECT_UUID, PROJECT_UUID, emptyList(), 30, Optional.of(30), System2.INSTANCE, emptySet()); + } + + private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String rootUuid, String projectUuid) { + return newConfigurationWith30Days(system2, rootUuid, projectUuid, Collections.emptySet()); } - private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String rootUuid, String projectUuid, String... disabledComponentUuids) { - return new PurgeConfiguration(rootUuid, projectUuid, emptyList(), 30, Optional.of(30), system2, asList(disabledComponentUuids)); + private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String rootUuid, String projectUuid, Set disabledComponentUuids) { + return new PurgeConfiguration(rootUuid, projectUuid, emptyList(), 30, Optional.of(30), system2, disabledComponentUuids); } } -- 2.39.5