From: Duarte Meneses Date: Wed, 18 Mar 2020 20:50:42 +0000 (-0500) Subject: SONAR-12691 Dont delete past measures of directories and files X-Git-Tag: 8.3.0.34182~76 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=4f5231d18118c6643c19f1d1259f1165b2764774;p=sonarqube.git SONAR-12691 Dont delete past measures of directories and files --- diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/purge/PurgeDatastoresStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/purge/PurgeDatastoresStepTest.java index 2a78a36a368..3a1fd3d8f71 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/purge/PurgeDatastoresStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/purge/PurgeDatastoresStepTest.java @@ -19,9 +19,7 @@ */ package org.sonar.ce.task.projectanalysis.purge; -import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; -import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.util.Arrays; import java.util.Collections; import java.util.function.Predicate; @@ -40,11 +38,11 @@ import org.sonar.ce.task.projectanalysis.component.ReportComponent; import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; import org.sonar.ce.task.projectanalysis.component.ViewsComponent; import org.sonar.ce.task.projectanalysis.step.BaseStepTest; +import org.sonar.ce.task.projectanalysis.util.WrapInSingleElementArray; import org.sonar.ce.task.step.ComputationStep; import org.sonar.ce.task.step.TestComputationStepContext; import org.sonar.db.DbClient; import org.sonar.server.project.Project; -import org.sonar.ce.task.projectanalysis.util.WrapInSingleElementArray; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -52,7 +50,6 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @RunWith(DataProviderRunner.class) @@ -91,41 +88,7 @@ public class PurgeDatastoresStepTest extends BaseStepTest { verify_call_purge_method_of_the_purge_task(project); } - - @DataProvider - public static Object[][] nonRootProjectComponentTypes() { - return dataproviderFromComponentTypeValues(input -> input.isReportType() && input != Component.Type.PROJECT); - } - - @Test - @UseDataProvider("nonRootProjectComponentTypes") - public void do_not_call_purge_method_of_the_purge_task_for_other_report_components(Component.Type type) { - Component component = ReportComponent.builder(type, 1).setUuid(PROJECT_UUID).setKey(PROJECT_KEY).build(); - - verify_do_not_call_purge_method_of_the_purge_task(component); - } - - @DataProvider - public static Object[][] nonRootViewsComponentTypes() { - return dataproviderFromComponentTypeValues(input -> input.isViewsType() && input != Component.Type.VIEW); - } - - @Test - @UseDataProvider("nonRootViewsComponentTypes") - public void do_not_call_purge_method_of_the_purge_task_for_other_views_components(Component.Type type) { - Component component = ViewsComponent.builder(type, PROJECT_KEY).setUuid(PROJECT_UUID).build(); - - verify_do_not_call_purge_method_of_the_purge_task(component); - } - - private void verify_do_not_call_purge_method_of_the_purge_task(Component component) { - treeRootHolder.setRoot(component); - - underTest.execute(new TestComputationStepContext()); - - verifyNoMoreInteractions(projectCleaner); - } - + private void verify_call_purge_method_of_the_purge_task(Component project) { treeRootHolder.setRoot(project); when(settingsRepository.getConfiguration()).thenReturn(new MapSettings().asConfig()); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java index 1cfa3e45f7f..b27aa2fcf14 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -25,7 +25,6 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import javax.annotation.Nullable; import org.sonar.api.utils.System2; import org.sonar.db.DbSession; @@ -55,10 +54,6 @@ class PurgeCommands { this(session, session.getMapper(PurgeMapper.class), profiler, system2); } - List selectSnapshotUuids(PurgeSnapshotQuery query) { - return purgeMapper.selectAnalysisIdsAndUuids(query).stream().map(IdUuidPair::getUuid).collect(Collectors.toList()); - } - List selectSnapshotIdUuids(PurgeSnapshotQuery query) { return purgeMapper.selectAnalysisIdsAndUuids(query); } @@ -312,24 +307,6 @@ class PurgeCommands { profiler.stop(); } - void deleteComponentMeasures(List analysisUuids, List componentUuids) { - if (analysisUuids.isEmpty() || componentUuids.isEmpty()) { - return; - } - - List> analysisUuidsPartitions = Lists.partition(analysisUuids, MAX_SNAPSHOTS_PER_QUERY); - List> componentUuidsPartitions = Lists.partition(componentUuids, MAX_RESOURCES_PER_QUERY); - - profiler.start("deleteComponentMeasures"); - for (List analysisUuidsPartition : analysisUuidsPartitions) { - for (List componentUuidsPartition : componentUuidsPartitions) { - purgeMapper.deleteComponentMeasures(analysisUuidsPartition, componentUuidsPartition); - } - } - session.commit(); - profiler.stop(); - } - void deleteFileSources(String rootUuid) { profiler.start("deleteFileSources (file_sources)"); purgeMapper.deleteFileSourcesByProjectUuid(rootUuid); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java index f3dc066dcab..5e936d88649 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeConfiguration.java @@ -20,14 +20,11 @@ package org.sonar.db.purge; import com.google.common.annotations.VisibleForTesting; -import java.util.Arrays; -import java.util.Collection; import java.util.Date; import java.util.Optional; import java.util.Set; import javax.annotation.CheckForNull; import org.sonar.api.config.Configuration; -import org.sonar.api.resources.Scopes; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; import org.sonar.core.config.PurgeConstants; @@ -36,17 +33,15 @@ public class PurgeConfiguration { private final String rootUuid; private final String projectUuid; - private final Collection scopesWithoutHistoricalData; private final int maxAgeInDaysOfClosedIssues; private final Optional maxAgeInDaysOfInactiveBranches; private final System2 system2; private final Set disabledComponentUuids; - public PurgeConfiguration(String rootUuid, String projectUuid, Collection scopesWithoutHistoricalData, int maxAgeInDaysOfClosedIssues, + public PurgeConfiguration(String rootUuid, String projectUuid, int maxAgeInDaysOfClosedIssues, Optional maxAgeInDaysOfInactiveBranches, System2 system2, Set disabledComponentUuids) { this.rootUuid = rootUuid; this.projectUuid = projectUuid; - this.scopesWithoutHistoricalData = scopesWithoutHistoricalData; this.maxAgeInDaysOfClosedIssues = maxAgeInDaysOfClosedIssues; this.system2 = system2; this.disabledComponentUuids = disabledComponentUuids; @@ -54,7 +49,7 @@ public class PurgeConfiguration { } public static PurgeConfiguration newDefaultPurgeConfiguration(Configuration config, String rootUuid, String projectUuid, Set disabledComponentUuids) { - return new PurgeConfiguration(rootUuid, projectUuid, Arrays.asList(Scopes.DIRECTORY, Scopes.FILE), config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES).get(), + return new PurgeConfiguration(rootUuid, projectUuid, config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES).get(), config.getInt(PurgeConstants.DAYS_BEFORE_DELETING_INACTIVE_BRANCHES_AND_PRS), System2.INSTANCE, disabledComponentUuids); } @@ -74,10 +69,6 @@ public class PurgeConfiguration { return projectUuid; } - public Collection getScopesWithoutHistoricalData() { - return scopesWithoutHistoricalData; - } - public Set getDisabledComponentUuids() { return disabledComponentUuids; } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java index 107a444bfd9..b88546e6a6e 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -35,10 +35,7 @@ import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.Dao; import org.sonar.db.DbSession; import org.sonar.db.component.BranchMapper; -import org.sonar.db.component.ComponentDao; import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.ComponentTreeQuery; -import org.sonar.db.component.ComponentTreeQuery.Strategy; import static java.util.Collections.emptyList; import static java.util.Optional.ofNullable; @@ -51,11 +48,9 @@ public class PurgeDao implements Dao { private static final Set QUALIFIERS_MODULE_SUBVIEW = ImmutableSet.of("BRC", "SVW"); private static final String SCOPE_PROJECT = "PRJ"; - private final ComponentDao componentDao; private final System2 system2; - public PurgeDao(ComponentDao componentDao, System2 system2) { - this.componentDao = componentDao; + public PurgeDao(System2 system2) { this.system2 = system2; } @@ -64,7 +59,6 @@ public class PurgeDao implements Dao { PurgeCommands commands = new PurgeCommands(session, mapper, profiler, system2); String rootUuid = conf.rootUuid(); deleteAbortedAnalyses(rootUuid, commands); - deleteDataOfComponentsWithoutHistoricalData(session, rootUuid, conf.getScopesWithoutHistoricalData(), commands); purgeAnalyses(commands, rootUuid); purgeDisabledComponents(commands, conf, listener); deleteOldClosedIssues(conf, mapper, listener); @@ -127,29 +121,6 @@ public class PurgeDao implements Dao { commands.deleteAbortedAnalyses(rootUuid); } - private void deleteDataOfComponentsWithoutHistoricalData(DbSession dbSession, String rootUuid, Collection scopesWithoutHistoricalData, PurgeCommands purgeCommands) { - if (scopesWithoutHistoricalData.isEmpty()) { - return; - } - - List analysisUuids = purgeCommands.selectSnapshotUuids( - new PurgeSnapshotQuery(rootUuid) - .setIslast(false) - .setNotPurged(true)); - List componentWithoutHistoricalDataUuids = componentDao - .selectDescendants( - dbSession, - ComponentTreeQuery.builder() - .setBaseUuid(rootUuid) - .setScopes(scopesWithoutHistoricalData) - .setStrategy(Strategy.LEAVES) - .build()) - .stream().map(ComponentDto::uuid) - .collect(MoreCollectors.toList()); - - purgeCommands.deleteComponentMeasures(analysisUuids, componentWithoutHistoricalDataUuids); - } - private static void deleteOldDisabledComponents(PurgeCommands commands, PurgeMapper mapper, String rootUuid) { List disabledComponentsWithoutIssue = mapper.selectDisabledComponentsWithoutIssues(rootUuid); commands.deleteDisabledComponentsWithoutIssues(disabledComponentsWithoutIssue); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java index 049cb8792b6..07052cbed3b 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -54,8 +54,6 @@ public interface PurgeMapper { void fullDeleteComponentMeasures(@Param("componentUuids") List componentUuids); - void deleteComponentMeasures(@Param("analysisUuids") List analysisUuids, @Param("componentUuids") List componentUuids); - List selectMetricIdsWithoutHistoricalData(); void deleteAnalysisWastedMeasures(@Param("analysisUuids") List analysisUuids, @Param("metricIds") List metricIds); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index 2d1dffdcf62..b2c7e568444 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -128,19 +128,6 @@ - - delete from project_measures - where - analysis_uuid in - - #{analysisUuid,jdbcType=VARCHAR} - - and component_uuid in - - #{componentUuid,jdbcType=VARCHAR} - - - delete from duplications_index where diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java index d3012972aad..964bdde2192 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeConfigurationTest.java @@ -22,13 +22,8 @@ package org.sonar.db.purge; import java.util.Date; import java.util.Optional; import org.junit.Test; -import org.sonar.api.config.PropertyDefinitions; -import org.sonar.api.config.internal.MapSettings; -import org.sonar.api.resources.Scopes; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; -import org.sonar.core.config.PurgeConstants; -import org.sonar.core.config.PurgeProperties; import static java.util.Collections.emptySet; import static org.assertj.core.api.Assertions.assertThat; @@ -36,10 +31,10 @@ import static org.assertj.core.api.Assertions.assertThat; public class PurgeConfigurationTest { @Test public void should_delete_all_closed_issues() { - PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptySet(), 0, Optional.empty(), System2.INSTANCE, emptySet()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", 0, Optional.empty(), System2.INSTANCE, emptySet()); assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); - conf = new PurgeConfiguration("root", "project", emptySet(), -1, Optional.empty(), System2.INSTANCE, emptySet()); + conf = new PurgeConfiguration("root", "project", -1, Optional.empty(), System2.INSTANCE, emptySet()); assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); } @@ -47,7 +42,7 @@ public class PurgeConfigurationTest { public void should_delete_only_old_closed_issues() { Date now = DateUtils.parseDate("2013-05-18"); - PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptySet(), 30, Optional.empty(), System2.INSTANCE, emptySet()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.empty(), System2.INSTANCE, emptySet()); Date toDate = conf.maxLiveDateOfClosedIssues(now); assertThat(toDate.getYear()).isEqualTo(113);// =2013 @@ -57,7 +52,7 @@ public class PurgeConfigurationTest { @Test public void should_have_empty_branch_purge_date() { - PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptySet(), 30, Optional.of(10), System2.INSTANCE, emptySet()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.of(10), System2.INSTANCE, emptySet()); assertThat(conf.maxLiveDateOfInactiveBranches()).isNotEmpty(); long tenDaysAgo = DateUtils.addDays(new Date(System2.INSTANCE.now()), -10).getTime(); assertThat(conf.maxLiveDateOfInactiveBranches().get().getTime()).isBetween(tenDaysAgo - 5000, tenDaysAgo + 5000); @@ -65,20 +60,8 @@ public class PurgeConfigurationTest { @Test public void should_calculate_branch_purge_date() { - PurgeConfiguration conf = new PurgeConfiguration("root", "project", emptySet(), 30, Optional.empty(), System2.INSTANCE, emptySet()); + PurgeConfiguration conf = new PurgeConfiguration("root", "project", 30, Optional.empty(), System2.INSTANCE, emptySet()); assertThat(conf.maxLiveDateOfInactiveBranches()).isEmpty(); } - @Test - public void delete_files_and_directories() { - MapSettings settings = new MapSettings(new PropertyDefinitions(PurgeProperties.all())); - settings.setProperty(PurgeConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES, 5); - Date now = new Date(); - - PurgeConfiguration underTest = PurgeConfiguration.newDefaultPurgeConfiguration(settings.asConfig(), "root", "project", emptySet()); - - assertThat(underTest.getScopesWithoutHistoricalData()) - .containsExactlyInAnyOrder(Scopes.DIRECTORY, Scopes.FILE); - assertThat(underTest.maxLiveDateOfClosedIssues(now)).isEqualTo(DateUtils.addDays(now, -5)); - } } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java index 67288494451..e30e5008d05 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java @@ -41,7 +41,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.sonar.api.issue.Issue; -import org.sonar.api.resources.Scopes; import org.sonar.api.utils.System2; import org.sonar.core.util.CloseableIterator; import org.sonar.core.util.UuidFactoryFast; @@ -85,7 +84,6 @@ import org.sonar.db.webhook.WebhookDto; import static com.google.common.base.MoreObjects.firstNonNull; import static java.time.ZoneOffset.UTC; import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; @@ -216,7 +214,7 @@ public class PurgeDaoTest { } @Test - public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() { + public void shouldDeleteNonHistoricalData() { MetricDto metricWithHistory = db.measures().insertMetric(t -> t.setDeleteHistoricalData(false)); MetricDto metricWithoutHistory = db.measures().insertMetric(t -> t.setDeleteHistoricalData(true)); ComponentDto project = db.components().insertPrivateProject(); @@ -234,8 +232,7 @@ public class PurgeDaoTest { db.measures().insertMeasure(otherProject, otherOldAnalysis, metricWithHistory); db.measures().insertMeasure(otherProject, otherOldAnalysis, metricWithoutHistory); - PurgeConfiguration conf = new PurgeConfiguration(project.uuid(), project.uuid(), asList(Scopes.DIRECTORY, Scopes.FILE), - 30, Optional.of(30), System2.INSTANCE, emptySet()); + PurgeConfiguration conf = new PurgeConfiguration(project.uuid(), project.uuid(), 30, Optional.of(30), System2.INSTANCE, emptySet()); underTest.purge(dbSession, conf, PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); @@ -1739,7 +1736,7 @@ public class PurgeDaoTest { } private static PurgeConfiguration newConfigurationWith30Days(String rootUuid) { - return new PurgeConfiguration(rootUuid, rootUuid, emptyList(), 30, Optional.of(30), System2.INSTANCE, emptySet()); + return new PurgeConfiguration(rootUuid, rootUuid, 30, Optional.of(30), System2.INSTANCE, emptySet()); } private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String rootUuid, String projectUuid) { @@ -1747,7 +1744,7 @@ public class PurgeDaoTest { } private static PurgeConfiguration newConfigurationWith30Days(System2 system2, String rootUuid, String projectUuid, Set disabledComponentUuids) { - return new PurgeConfiguration(rootUuid, projectUuid, emptyList(), 30, Optional.of(30), system2, disabledComponentUuids); + return new PurgeConfiguration(rootUuid, projectUuid, 30, Optional.of(30), system2, disabledComponentUuids); } private Stream uuidsOfAnalysesOfRoot(ComponentDto rootComponent) {