From c8f5214778ac4c3e16e35d9c47a055a2667fb39a Mon Sep 17 00:00:00 2001 From: Sébastien Lesaint Date: Thu, 30 Jun 2016 09:57:13 +0200 Subject: SONAR-7705 purge: delete measures of FILE and DIR via analysis uuid --- .../java/org/sonar/db/purge/PurgeCommands.java | 39 ++++++++---- .../src/main/java/org/sonar/db/purge/PurgeDao.java | 71 +++++++++++++++++----- .../main/java/org/sonar/db/purge/PurgeMapper.java | 3 +- .../resources/org/sonar/db/purge/PurgeMapper.xml | 13 ++++ .../test/java/org/sonar/db/purge/PurgeDaoTest.java | 2 +- ...eHistoricalDataOfDirectoriesAndFiles-result.xml | 6 +- ...ldDeleteHistoricalDataOfDirectoriesAndFiles.xml | 6 +- 7 files changed, 106 insertions(+), 34 deletions(-) (limited to 'sonar-db') diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java index c959a949795..3273834f56a 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -175,10 +175,28 @@ class PurgeCommands { profiler.stop(); } + public void deleteComponentMeasures(List analysisUuids, List componentUuids) { + if (analysisUuids.isEmpty() || componentUuids.isEmpty()) { + return; + } + + List> analysisUuidsPartitions = Lists.partition(analysisUuids, MAX_SNAPSHOTS_PER_QUERY); + List> componentUuidsPartitions = Lists.partition(componentUuids, MAX_RESOURCES_PER_QUERY); + + profiler.start("deleteComponentMeasures"); + for (List analysisUuidsPartition : analysisUuidsPartitions) { + for (List componentUuidsPartition : componentUuidsPartitions) { + purgeMapper.deleteComponentMeasures(analysisUuidsPartition, componentUuidsPartition); + } + } + session.commit(); + profiler.stop(); + } + void deleteAnalyses(PurgeSnapshotQuery... queries) { List snapshotIds = from(asList(queries)) - .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids) - .toList(); + .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids) + .toList(); deleteAnalyses(snapshotIds); } @@ -209,23 +227,22 @@ class PurgeCommands { void purgeSnapshots(PurgeSnapshotQuery... queries) { // use LinkedHashSet to keep order by remove duplicated ids LinkedHashSet snapshotIds = Sets.newLinkedHashSet(from(asList(queries)) - .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids)); + .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids)); purgeSnapshots(snapshotIds); } @VisibleForTesting protected void purgeSnapshots(Iterable snapshotIdUuidPairs) { // note that events are not deleted - List> snapshotIdsPartition = Lists.partition(IdUuidPairs.ids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY); - List> snapshotUuidsPartition = Lists.partition(IdUuidPairs.uuids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY); + List> snapshotIdsPartitions = Lists.partition(IdUuidPairs.ids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY); + List> snapshotUuidsPartitions = Lists.partition(IdUuidPairs.uuids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY); - deleteSnapshotDuplications(snapshotUuidsPartition); + deleteSnapshotDuplications(snapshotUuidsPartitions); profiler.start("deleteSnapshotWastedMeasures (project_measures)"); List metricIdsWithoutHistoricalData = purgeMapper.selectMetricIdsWithoutHistoricalData(); - for (List partSnapshotIds : snapshotIdsPartition) { - purgeMapper.deleteSnapshotWastedMeasures(partSnapshotIds, metricIdsWithoutHistoricalData); - } + snapshotIdsPartitions.stream() + .forEach(snapshotIdsPartition -> purgeMapper.deleteSnapshotWastedMeasures(snapshotIdsPartition, metricIdsWithoutHistoricalData)); session.commit(); profiler.stop(); @@ -235,9 +252,9 @@ class PurgeCommands { profiler.stop(); } - private void deleteSnapshotDuplications(List> snapshotUuidsPartition) { + private void deleteSnapshotDuplications(List> snapshotUuidsPartitions) { profiler.start("deleteSnapshotDuplications (duplications_index)"); - snapshotUuidsPartition.forEach(purgeMapper::deleteSnapshotDuplications); + snapshotUuidsPartitions.forEach(purgeMapper::deleteSnapshotDuplications); session.commit(); profiler.stop(); } diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java index 96df1ac2db1..79fdd11aed4 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -21,17 +21,21 @@ package org.sonar.db.purge; import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; -import org.apache.commons.lang.ArrayUtils; import org.apache.ibatis.session.SqlSession; import org.sonar.api.resources.Scopes; import org.sonar.api.utils.System2; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; +import org.sonar.core.util.stream.GuavaCollectors; import org.sonar.db.Dao; import org.sonar.db.DbSession; +import org.sonar.db.component.ComponentDao; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ComponentTreeQuery; import org.sonar.db.component.ResourceDao; import org.sonar.db.component.ResourceDto; @@ -45,12 +49,15 @@ import static org.sonar.db.DatabaseUtils.executeLargeInputs; public class PurgeDao implements Dao { private static final Logger LOG = Loggers.get(PurgeDao.class); private static final String[] UNPROCESSED_STATUS = new String[] {"U"}; + private static final List UUID_FIELD_SORT = Collections.singletonList("uuid"); private final ResourceDao resourceDao; + private final ComponentDao componentDao; private final System2 system2; - public PurgeDao(ResourceDao resourceDao, System2 system2) { + public PurgeDao(ResourceDao resourceDao, ComponentDao componentDao, System2 system2) { this.resourceDao = resourceDao; + this.componentDao = componentDao; this.system2 = system2; } @@ -58,10 +65,12 @@ public class PurgeDao implements Dao { PurgeMapper mapper = session.getMapper(PurgeMapper.class); PurgeCommands commands = new PurgeCommands(session, mapper, profiler); deleteAbortedAnalyses(conf.rootProjectIdUuid().getUuid(), commands); + deleteDataOfComponentsWithoutHistoricalData(session, conf.rootProjectIdUuid().getUuid(), conf.scopesWithoutHistoricalData(), commands); + // retrieve all nodes in the tree (including root) with scope=PROJECT List projects = getProjects(conf.rootProjectIdUuid().getId(), session); for (ResourceDto project : projects) { LOG.debug("-> Clean " + project.getLongName() + " [id=" + project.getId() + "]"); - purge(project, conf.scopesWithoutHistoricalData(), commands); + purge(project.getUuid(), commands); } for (ResourceDto project : projects) { disableOrphanResources(project, session, mapper, listener); @@ -92,21 +101,53 @@ public class PurgeDao implements Dao { commands.deleteAnalyses(query); } - private static void purge(ResourceDto project, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) { + private void deleteDataOfComponentsWithoutHistoricalData(DbSession dbSession, String rootUuid, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) { + if (scopesWithoutHistoricalData.length == 0) { + return; + } + + List analysisUuids = purgeCommands.selectSnapshotUuids( + PurgeSnapshotQuery.create() + .setComponentUuid(rootUuid) + .setIslast(false) + .setNotPurged(true)); + List componentWithoutHistoricalDataUuids = componentDao + .selectDescendants( + dbSession, + newComponentTreeQuery() + .setBaseUuid(rootUuid) + .setQualifiers(Arrays.asList(scopesWithoutHistoricalData)) + .build()) + .stream().map(ComponentDto::uuid) + .collect(GuavaCollectors.toList()); + + purgeCommands.deleteComponentMeasures(analysisUuids, componentWithoutHistoricalDataUuids); + // FIXME remove this when cardinality of snapshots has been changed + for (String componentUuid : componentWithoutHistoricalDataUuids) { + purgeCommands.deleteSnapshots(PurgeSnapshotQuery.create() + .setIslast(false) + .setComponentUuid(componentUuid)); + } + } + + /** + * Creates a new ComponentTreeQuery.Builder with properties that don't matter here but are mandatory populated. + */ + private static ComponentTreeQuery.Builder newComponentTreeQuery() { + return ComponentTreeQuery.builder() + .setPage(1) + .setPageSize(Integer.MAX_VALUE) + .setSortFields(UUID_FIELD_SORT); + } + + private static void purge(String componentUuid, PurgeCommands purgeCommands) { List projectSnapshotUuids = purgeCommands.selectSnapshotUuids( - PurgeSnapshotQuery.create() - .setComponentUuid(project.getUuid()) - .setIslast(false) - .setNotPurged(true)); + PurgeSnapshotQuery.create() + .setComponentUuid(componentUuid) + .setIslast(false) + .setNotPurged(true)); for (String snapshotUuid : projectSnapshotUuids) { LOG.debug("<- Clean analysis " + snapshotUuid); - if (!ArrayUtils.isEmpty(scopesWithoutHistoricalData)) { - PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setScopes(scopesWithoutHistoricalData) - .setAnalysisUuid(snapshotUuid); - purgeCommands.deleteSnapshots(query); - } // must be executed at the end for reentrance purgeCommands.purgeSnapshots( diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java index fc2ec93eff3..5ebe7ded301 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -46,6 +46,8 @@ public interface PurgeMapper { void deleteSnapshotMeasures(@Param("snapshotIds") List snapshotIds); + void deleteComponentMeasures(@Param("analysisUuids") List analysisUuids, @Param("componentUuids") List componentUuids); + List selectMetricIdsWithoutHistoricalData(); void deleteSnapshotWastedMeasures(@Param("snapshotIds") List snapshotIds, @Param("mids") List metricIds); @@ -95,5 +97,4 @@ public interface PurgeMapper { void deleteFileSourcesByUuid(String fileUuid); void deleteCeActivityByProjectUuid(String projectUuid); - } diff --git a/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index a1bcc631507..b087ab53da1 100644 --- a/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -109,6 +109,19 @@ + + delete from project_measures + where + analysis_uuid in + + #{analysisUuid} + + and component_uuid in + + #{componentUuid} + + + delete from duplications_index where analysis_uuid in diff --git a/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java index 154dda8bc0a..05aa18219a0 100644 --- a/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java +++ b/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java @@ -84,7 +84,7 @@ public class PurgeDaoTest { @Test public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() { dbTester.prepareDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml"); - underTest.purge(dbSession, new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "1"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, 30), PurgeListener.EMPTY, + underTest.purge(dbSession, new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "ABCD"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, 30), PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); dbTester.assertDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml", "projects", "snapshots"); diff --git a/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml b/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml index 2b56c7bb883..4c119f1b43d 100644 --- a/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml +++ b/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml @@ -8,7 +8,7 @@ What has been changed : purge_status=1 on snapshot 4 (PRJ) and snapshots 5 and 6