From: Julien HENRY Date: Wed, 12 Jun 2013 10:44:58 +0000 (+0200) Subject: SONAR-4368 Highly improve performance of the project deletion operation X-Git-Tag: 3.7~505 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=e534ce920a8b372156fe6510c14f93ab39f009ce;p=sonarqube.git SONAR-4368 Highly improve performance of the project deletion operation --- diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeCommands.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeCommands.java index dc7ee6e0454..e37b0fef704 100644 --- a/sonar-core/src/main/java/org/sonar/core/purge/PurgeCommands.java +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeCommands.java @@ -21,12 +21,15 @@ package org.sonar.core.purge; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.apache.ibatis.session.SqlSession; import java.util.List; class PurgeCommands { private static final int MAX_CHARACTERISTICS_PER_QUERY = 1000; + private static final int MAX_SNAPSHOTS_PER_QUERY = 1000; + private static final int MAX_RESOURCES_PER_QUERY = 1000; private final SqlSession session; private final PurgeMapper purgeMapper; @@ -48,103 +51,104 @@ class PurgeCommands { } void deleteResources(List resourceIds) { + List> resourceIdsPartition = Lists.partition(resourceIds, MAX_RESOURCES_PER_QUERY); // Note : do not merge the delete statements into a single loop of resource ids. It's // voluntarily grouped by tables in order to benefit from JDBC batch mode. // Batch requests can only relate to the same PreparedStatement. - for (Long resourceId : resourceIds) { - deleteSnapshots(PurgeSnapshotQuery.create().setResourceId(resourceId)); + for (List partResourceIds : resourceIdsPartition) { + deleteSnapshots(purgeMapper.selectSnapshotIdsByResource(partResourceIds)); } // possible missing optimization: filter requests according to resource scope profiler.start("deleteResourceLinks (project_links)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceLinks(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceLinks(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceProperties (properties)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceProperties(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceProperties(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceIndex (resource_index)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceIndex(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceIndex(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceGroupRoles (group_roles)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceGroupRoles(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceGroupRoles(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceUserRoles (user_roles)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceUserRoles(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceUserRoles(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceManualMeasures (manual_measures)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceManualMeasures(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceManualMeasures(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceIssueChanges (issue_changes)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceIssueChanges(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceIssueChanges(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceIssues (issues)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceIssues(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceIssues(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceActionPlans (action_plans)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceActionPlans(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceActionPlans(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceEvents (events)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceEvents(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceEvents(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResourceGraphs (graphs)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResourceGraphs(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResourceGraphs(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteResource (projects)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteResource(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteResource(partResourceIds); } session.commit(); profiler.stop(); profiler.start("deleteAuthors (authors)"); - for (Long resourceId : resourceIds) { - purgeMapper.deleteAuthors(resourceId); + for (List partResourceIds : resourceIdsPartition) { + purgeMapper.deleteAuthors(partResourceIds); } session.commit(); profiler.stop(); @@ -156,40 +160,42 @@ class PurgeCommands { private void deleteSnapshots(final List snapshotIds) { - deleteSnapshotDependencies(snapshotIds); + List> snapshotIdsPartition = Lists.partition(snapshotIds, MAX_SNAPSHOTS_PER_QUERY); - deleteSnapshotDuplications(snapshotIds); + deleteSnapshotDependencies(snapshotIdsPartition); + + deleteSnapshotDuplications(snapshotIdsPartition); profiler.start("deleteSnapshotEvents (events)"); - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshotEvents(snapshotId); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshotEvents(partSnapshotIds); } session.commit(); profiler.stop(); profiler.start("deleteSnapshotMeasureData (measure_data)"); - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshotMeasureData(snapshotId); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshotMeasureData(partSnapshotIds); } session.commit(); profiler.stop(); profiler.start("deleteSnapshotMeasures (project_measures)"); - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshotMeasures(snapshotId); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshotMeasures(partSnapshotIds); } session.commit(); profiler.stop(); - deleteSnapshotSources(snapshotIds); + deleteSnapshotSources(snapshotIdsPartition); - deleteSnapshotGraphs(snapshotIds); + deleteSnapshotGraphs(snapshotIdsPartition); - deleteSnapshotData(snapshotIds); + deleteSnapshotData(snapshotIdsPartition); profiler.start("deleteSnapshot (snapshots)"); - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshot(snapshotId); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshot(partSnapshotIds); } session.commit(); profiler.stop(); @@ -201,22 +207,23 @@ class PurgeCommands { private void purgeSnapshots(final List snapshotIds) { // note that events are not deleted + List> snapshotIdsPartition = Lists.partition(snapshotIds, MAX_SNAPSHOTS_PER_QUERY); - deleteSnapshotDependencies(snapshotIds); + deleteSnapshotDependencies(snapshotIdsPartition); - deleteSnapshotDuplications(snapshotIds); + deleteSnapshotDuplications(snapshotIdsPartition); - deleteSnapshotSources(snapshotIds); + deleteSnapshotSources(snapshotIdsPartition); - deleteSnapshotGraphs(snapshotIds); + deleteSnapshotGraphs(snapshotIdsPartition); - deleteSnapshotData(snapshotIds); + deleteSnapshotData(snapshotIdsPartition); profiler.start("deleteSnapshotWastedMeasures (project_measures)"); List metricIdsWithoutHistoricalData = purgeMapper.selectMetricIdsWithoutHistoricalData(); if (!metricIdsWithoutHistoricalData.isEmpty()) { - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshotWastedMeasures(snapshotId, metricIdsWithoutHistoricalData); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshotWastedMeasures(partSnapshotIds, metricIdsWithoutHistoricalData); } session.commit(); } @@ -225,10 +232,10 @@ class PurgeCommands { profiler.start("deleteSnapshotMeasuresOnCharacteristics (project_measures)"); List characteristicIds = purgeMapper.selectCharacteristicIdsToPurge(); if (!characteristicIds.isEmpty()) { - for (Long snapshotId : snapshotIds) { + for (List partSnapshotIds : snapshotIdsPartition) { // SONAR-3641 We cannot process all characteristics at once for (List ids : Iterables.partition(characteristicIds, MAX_CHARACTERISTICS_PER_QUERY)) { - purgeMapper.deleteSnapshotMeasuresOnCharacteristics(snapshotId, ids); + purgeMapper.deleteSnapshotMeasuresOnCharacteristics(partSnapshotIds, ids); } } session.commit(); @@ -243,46 +250,46 @@ class PurgeCommands { profiler.stop(); } - private void deleteSnapshotData(final List snapshotIds) { + private void deleteSnapshotData(final List> snapshotIdsPartition) { profiler.start("deleteSnapshotData (snapshot_data)"); - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshotData(snapshotId); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshotData(partSnapshotIds); } session.commit(); profiler.stop(); } - private void deleteSnapshotGraphs(final List snapshotIds) { + private void deleteSnapshotGraphs(final List> snapshotIdsPartition) { profiler.start("deleteSnapshotGraphs (graphs)"); - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshotGraphs(snapshotId); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshotGraphs(partSnapshotIds); } session.commit(); profiler.stop(); } - private void deleteSnapshotSources(final List snapshotIds) { + private void deleteSnapshotSources(final List> snapshotIdsPartition) { profiler.start("deleteSnapshotSource (snapshot_sources)"); - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshotSource(snapshotId); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshotSource(partSnapshotIds); } session.commit(); profiler.stop(); } - private void deleteSnapshotDuplications(final List snapshotIds) { + private void deleteSnapshotDuplications(final List> snapshotIdsPartition) { profiler.start("deleteSnapshotDuplications (duplications_index)"); - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshotDuplications(snapshotId); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshotDuplications(partSnapshotIds); } session.commit(); profiler.stop(); } - private void deleteSnapshotDependencies(final List snapshotIds) { + private void deleteSnapshotDependencies(final List> snapshotIdsPartition) { profiler.start("deleteSnapshotDependencies (dependencies)"); - for (Long snapshotId : snapshotIds) { - purgeMapper.deleteSnapshotDependencies(snapshotId); + for (List partSnapshotIds : snapshotIdsPartition) { + purgeMapper.deleteSnapshotDependencies(partSnapshotIds); } session.commit(); profiler.stop(); diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java index 9c848aabf23..fd8c0cfecc2 100644 --- a/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java @@ -31,6 +31,7 @@ import org.sonar.core.persistence.MyBatis; import org.sonar.core.resource.ResourceDao; import org.sonar.core.resource.ResourceDto; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; @@ -50,7 +51,6 @@ public class PurgeDao { this.profiler = profiler; } - public PurgeDao purge(PurgeConfiguration conf) { SqlSession session = mybatis.openBatchSession(); PurgeMapper mapper = session.getMapper(PurgeMapper.class); @@ -83,32 +83,32 @@ public class PurgeDao { if (hasAbortedBuilds(project.getId(), commands)) { LOG.info("<- Delete aborted builds"); PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setStatus(new String[]{"U"}) - .setRootProjectId(project.getId()); + .setIslast(false) + .setStatus(new String[] {"U"}) + .setRootProjectId(project.getId()); commands.deleteSnapshots(query); } } private boolean hasAbortedBuilds(Long projectId, PurgeCommands commands) { PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setStatus(new String[]{"U"}) - .setResourceId(projectId); + .setIslast(false) + .setStatus(new String[] {"U"}) + .setResourceId(projectId); return !commands.selectSnapshotIds(query).isEmpty(); } private void purge(ResourceDto project, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) { List projectSnapshotIds = purgeCommands.selectSnapshotIds( - PurgeSnapshotQuery.create().setResourceId(project.getId()).setIslast(false).setNotPurged(true) - ); + PurgeSnapshotQuery.create().setResourceId(project.getId()).setIslast(false).setNotPurged(true) + ); for (final Long projectSnapshotId : projectSnapshotIds) { LOG.info("<- Clean snapshot " + projectSnapshotId); if (!ArrayUtils.isEmpty(scopesWithoutHistoricalData)) { PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setScopes(scopesWithoutHistoricalData) - .setRootSnapshotId(projectSnapshotId); + .setIslast(false) + .setScopes(scopesWithoutHistoricalData) + .setRootSnapshotId(projectSnapshotId); purgeCommands.deleteSnapshots(query); } @@ -170,7 +170,7 @@ public class PurgeDao { @VisibleForTesting void disableResource(long resourceId, PurgeMapper mapper) { - mapper.deleteResourceIndex(resourceId); + mapper.deleteResourceIndex(Arrays.asList(resourceId)); mapper.setSnapshotIsLastToFalse(resourceId); mapper.disableResource(resourceId); } diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java index 66b1550780e..d7e3c2d034f 100644 --- a/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java @@ -22,6 +22,7 @@ package org.sonar.core.purge; import org.apache.ibatis.annotations.Param; import javax.annotation.Nullable; + import java.util.Date; import java.util.List; @@ -29,61 +30,65 @@ public interface PurgeMapper { List selectSnapshotIds(PurgeSnapshotQuery query); + List selectSnapshotIdsByResource(@Param("resourceIds") List resourceIds); + List selectProjectIdsByRootId(long rootResourceId); - void deleteSnapshot(long snapshotId); + void deleteSnapshot(@Param("snapshotIds") List snapshotIds); - void deleteSnapshotDependencies(long snapshotId); + void deleteSnapshotDependencies(@Param("snapshotIds") List snapshotIds); - void deleteSnapshotDuplications(long snapshotId); + void deleteSnapshotDuplications(@Param("snapshotIds") List snapshotIds); - void deleteSnapshotEvents(long snapshotId); + void deleteSnapshotEvents(@Param("snapshotIds") List snapshotIds); - void deleteSnapshotMeasures(long snapshotId); + void deleteSnapshotMeasures(@Param("snapshotIds") List snapshotIds); - void deleteSnapshotMeasureData(long snapshotId); + void deleteSnapshotMeasureData(@Param("snapshotIds") List snapshotIds); - void deleteSnapshotSource(long snapshotId); + void deleteSnapshotSource(@Param("snapshotIds") List snapshotIds); - void deleteSnapshotGraphs(long snapshotId); + void deleteSnapshotGraphs(@Param("snapshotIds") List snapshotIds); + + void deleteSnapshotData(@Param("snapshotIds") List snapshotIds); List selectMetricIdsWithoutHistoricalData(); List selectCharacteristicIdsToPurge(); - void deleteSnapshotWastedMeasures(@Param("sid") long snapshotId, @Param("mids") List metricIds); + void deleteSnapshotWastedMeasures(@Param("snapshotIds") List snapshotIds, @Param("mids") List metricIds); - void deleteSnapshotMeasuresOnCharacteristics(@Param("sid") long snapshotId, @Param("cids") List characteristicIds); + void deleteSnapshotMeasuresOnCharacteristics(@Param("snapshotIds") List snapshotIds, @Param("cids") List characteristicIds); void updatePurgeStatusToOne(long snapshotId); void disableResource(long resourceId); - void deleteResourceIndex(long resourceId); + void deleteResourceIndex(@Param("resourceIds") List resourceIds); void deleteEvent(long eventId); void setSnapshotIsLastToFalse(long resourceId); - void deleteResourceLinks(long resourceId); + void deleteResourceLinks(@Param("resourceIds") List resourceIds); - void deleteResourceProperties(long resourceId); + void deleteResourceProperties(@Param("resourceIds") List resourceIds); - void deleteResource(long resourceId); + void deleteResource(@Param("resourceIds") List resourceIds); - void deleteResourceGroupRoles(long resourceId); + void deleteResourceGroupRoles(@Param("resourceIds") List resourceIds); - void deleteResourceUserRoles(long resourceId); + void deleteResourceUserRoles(@Param("resourceIds") List resourceIds); - void deleteResourceManualMeasures(long resourceId); + void deleteResourceManualMeasures(@Param("resourceIds") List resourceIds); - void deleteResourceEvents(long resourceId); + void deleteResourceEvents(@Param("resourceIds") List resourceIds); - void deleteResourceActionPlans(long resourceId); + void deleteResourceActionPlans(@Param("resourceIds") List resourceIds); - void deleteResourceGraphs(long resourceId); + void deleteResourceGraphs(@Param("resourceIds") List resourceIds); - void deleteAuthors(long developerId); + void deleteAuthors(@Param("resourceIds") List resourceIds); List selectPurgeableSnapshotsWithEvents(long resourceId); @@ -91,11 +96,9 @@ public interface PurgeMapper { List selectResourceIdsByRootId(long rootProjectId); - void deleteSnapshotData(long snapshotId); - - void deleteResourceIssueChanges(long resourceId); + void deleteResourceIssueChanges(@Param("resourceIds") List resourceIds); - void deleteResourceIssues(long resourceId); + void deleteResourceIssues(@Param("resourceIds") List resourceIds); void deleteOldClosedIssueChanges(@Param("rootProjectId") long rootProjectId, @Nullable @Param("toDate") Date toDate); diff --git a/sonar-core/src/main/resources/org/sonar/core/purge/PurgeMapper.xml b/sonar-core/src/main/resources/org/sonar/core/purge/PurgeMapper.xml index 936acac2cdf..ae75f3c77d6 100644 --- a/sonar-core/src/main/resources/org/sonar/core/purge/PurgeMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/purge/PurgeMapper.xml @@ -47,6 +47,16 @@ + + - - delete from project_measures where snapshot_id=#{id} + + delete from project_measures where snapshot_id in + + #{snapshotId} + - - delete from measure_data where snapshot_id=#{id} + + delete from measure_data where snapshot_id in + + #{snapshotId} + - - delete from snapshot_sources where snapshot_id=#{id} + + delete from snapshot_sources where snapshot_id in + + #{snapshotId} + - - delete from graphs where snapshot_id=#{id} + + delete from graphs where snapshot_id in + + #{snapshotId} + - - delete from dependencies where from_snapshot_id=#{id} or to_snapshot_id=#{id} or project_snapshot_id=#{id} + + delete from dependencies where from_snapshot_id in + + #{snapshotId} + + or to_snapshot_id in + + #{snapshotId} + + or project_snapshot_id in + + #{snapshotId} + - - delete from duplications_index where snapshot_id=#{id} + + delete from duplications_index where snapshot_id in + + #{snapshotId} + - - delete from events where snapshot_id=#{id} + + delete from events where snapshot_id in + + #{snapshotId} + - - delete from snapshots where id=#{id} + + delete from snapshots where id in + + #{snapshotId} + - delete from project_measures where snapshot_id=#{sid} and + delete from project_measures where snapshot_id in + + #{snapshotId} + + and (rule_id is not null or person_id is not null or metric_id in #{mid} ) - delete from project_measures where snapshot_id=#{sid} + delete from project_measures where snapshot_id in + + #{snapshotId} + and ( characteristic_id=#{cid} ) @@ -138,81 +187,133 @@ update projects set enabled=${_false} where id=#{id} - - delete from resource_index where resource_id=#{id} + + delete from resource_index where resource_id in + + #{resourceId} + - - delete from events where id=#{id} + + delete from events where id in + + #{resourceId} + - - delete from project_links where project_id=#{id} + + delete from project_links where project_id in + + #{resourceId} + - - delete from properties where resource_id=#{id} + + delete from properties where resource_id in + + #{resourceId} + - - delete from projects where id=#{id} + + delete from projects where id in + + #{resourceId} + - - delete from group_roles where resource_id=#{id} + + delete from group_roles where resource_id in + + #{resourceId} + - - delete from user_roles where resource_id=#{id} + + delete from user_roles where resource_id in + + #{resourceId} + - - delete from manual_measures where resource_id=#{id} + + delete from manual_measures where resource_id in + + #{resourceId} + - - delete from events where resource_id=#{id} + + delete from events where resource_id in + + #{resourceId} + - - delete from action_plans where project_id=#{id} + + delete from action_plans where project_id in + + #{resourceId} + - - delete from graphs where resource_id=#{id} + + delete from graphs where resource_id in + + #{resourceId} + - - delete from authors where person_id=#{id} + + delete from authors where person_id in + + #{resourceId} + update snapshots set islast=${_false} where project_id=#{id} - - delete from snapshot_data where snapshot_id=#{id} + + delete from snapshot_data where snapshot_id in + + #{snapshotId} + - + delete from issue_changes ic - where exists (select * from issues i where i.kee=ic.issue_key and i.component_id=#{id}) + where exists (select * from issues i where i.kee=ic.issue_key and i.component_id in + + #{resourceId} + + ) - + delete issue_changes from issue_changes inner join issues on issue_changes.issue_key=issues.kee - where issues.component_id=#{id} + where issues.component_id in + + #{resourceId} + - - delete ic from issue_changes as ic, issues as i where ic.issue_key=i.kee and i.component_id=#{id} + + delete ic from issue_changes as ic, issues as i where ic.issue_key=i.kee and i.component_id in + + #{resourceId} + - - delete from issues where component_id=#{id} + + delete from issues where component_id in + + #{resourceId} +