From: Sébastien Lesaint Date: Thu, 21 Jan 2016 12:41:43 +0000 (+0100) Subject: SONAR-7130 group deletion and purges of snapshots all together X-Git-Tag: 5.4-M10~21 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=b2ffd22edae8c5ab09c8a7ac074af68351e46e56;p=sonarqube.git SONAR-7130 group deletion and purges of snapshots all together avoid doing select requests between deletion/purges of snapshots --- diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java index 1d5111a6a01..806703ebf6a 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -20,10 +20,18 @@ package org.sonar.db.purge; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.LinkedHashSet; import java.util.List; +import javax.annotation.Nullable; import org.apache.ibatis.session.SqlSession; +import static com.google.common.collect.FluentIterable.from; +import static java.util.Arrays.asList; + class PurgeCommands { private static final int MAX_SNAPSHOTS_PER_QUERY = 1000; @@ -32,6 +40,13 @@ class PurgeCommands { private final SqlSession session; private final PurgeMapper purgeMapper; private final PurgeProfiler profiler; + private final Function> purgeSnapshotQueryToSnapshotIds = new Function>() { + @Nullable + @Override + public Iterable apply(PurgeSnapshotQuery query) { + return purgeMapper.selectSnapshotIds(query); + } + }; PurgeCommands(SqlSession session, PurgeMapper purgeMapper, PurgeProfiler profiler) { this.session = session; @@ -146,8 +161,10 @@ class PurgeCommands { profiler.stop(); } - void deleteSnapshots(final PurgeSnapshotQuery query) { - deleteSnapshots(purgeMapper.selectSnapshotIds(query)); + void deleteSnapshots(PurgeSnapshotQuery... queries) { + List snapshotIds = from(asList(queries)) + .transformAndConcat(purgeSnapshotQueryToSnapshotIds).toList(); + deleteSnapshots(snapshotIds); } @VisibleForTesting @@ -179,14 +196,16 @@ class PurgeCommands { profiler.stop(); } - void purgeSnapshots(final PurgeSnapshotQuery query) { - purgeSnapshots(purgeMapper.selectSnapshotIds(query)); + void purgeSnapshots(PurgeSnapshotQuery... queries) { + // use LinkedHashSet to keep order by remove duplicated ids + LinkedHashSet snapshotIds = Sets.newLinkedHashSet(from(asList(queries)).transformAndConcat(purgeSnapshotQueryToSnapshotIds)); + purgeSnapshots(snapshotIds); } @VisibleForTesting - protected void purgeSnapshots(final List snapshotIds) { + protected void purgeSnapshots(Iterable snapshotIds) { // note that events are not deleted - List> snapshotIdsPartition = Lists.partition(snapshotIds, MAX_SNAPSHOTS_PER_QUERY); + Iterable> snapshotIdsPartition = Iterables.partition(snapshotIds, MAX_SNAPSHOTS_PER_QUERY); deleteSnapshotDuplications(snapshotIdsPartition); @@ -206,7 +225,7 @@ class PurgeCommands { profiler.stop(); } - private void deleteSnapshotDuplications(final List> snapshotIdsPartition) { + private void deleteSnapshotDuplications(Iterable> snapshotIdsPartition) { profiler.start("deleteSnapshotDuplications (duplications_index)"); for (List partSnapshotIds : snapshotIdsPartition) { purgeMapper.deleteSnapshotDuplications(partSnapshotIds); diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java index d1c206a7452..a0d89235131 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -46,6 +46,8 @@ import static org.sonar.api.utils.DateUtils.dateToLong; */ public class PurgeDao implements Dao { private static final Logger LOG = Loggers.get(PurgeDao.class); + private static final String[] UNPROCESSED_STATUS = new String[] {"U"}; + private final MyBatis mybatis; private final ResourceDao resourceDao; private final System2 system2; @@ -92,7 +94,7 @@ public class PurgeDao implements Dao { LOG.debug("<- Delete aborted builds"); PurgeSnapshotQuery query = PurgeSnapshotQuery.create() .setIslast(false) - .setStatus(new String[] {"U"}) + .setStatus(UNPROCESSED_STATUS) .setRootProjectId(project.getId()); commands.deleteSnapshots(query); } @@ -114,11 +116,10 @@ public class PurgeDao implements Dao { purgeCommands.deleteSnapshots(query); } - PurgeSnapshotQuery query = PurgeSnapshotQuery.create().setRootSnapshotId(projectSnapshotId).setNotPurged(true); - purgeCommands.purgeSnapshots(query); - // must be executed at the end for reentrance - purgeCommands.purgeSnapshots(PurgeSnapshotQuery.create().setId(projectSnapshotId).setNotPurged(true)); + purgeCommands.purgeSnapshots( + PurgeSnapshotQuery.create().setRootSnapshotId(projectSnapshotId).setNotPurged(true), + PurgeSnapshotQuery.create().setId(projectSnapshotId).setNotPurged(true)); } } @@ -186,15 +187,15 @@ public class PurgeDao implements Dao { public PurgeDao deleteSnapshots(PurgeSnapshotQuery query, PurgeProfiler profiler) { final DbSession session = mybatis.openSession(true); try { - return deleteSnapshots(query, session, profiler); + return deleteSnapshots(session, profiler, query); } finally { MyBatis.closeQuietly(session); } } - public PurgeDao deleteSnapshots(PurgeSnapshotQuery query, DbSession session, PurgeProfiler profiler) { - new PurgeCommands(session, profiler).deleteSnapshots(query); + public PurgeDao deleteSnapshots(DbSession session, PurgeProfiler profiler, PurgeSnapshotQuery... queries) { + new PurgeCommands(session, profiler).deleteSnapshots(queries); return this; } diff --git a/sonar-db/src/main/java/org/sonar/db/purge/period/DefaultPeriodCleaner.java b/sonar-db/src/main/java/org/sonar/db/purge/period/DefaultPeriodCleaner.java index fd34d37476e..8d41e91fbe4 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/period/DefaultPeriodCleaner.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/period/DefaultPeriodCleaner.java @@ -60,8 +60,10 @@ public class DefaultPeriodCleaner { private void delete(List snapshots, DbSession session) { for (PurgeableSnapshotDto snapshot : snapshots) { LOG.debug("<- Delete snapshot: {} [{}]", DateUtils.formatDateTime(snapshot.getDate()), snapshot.getSnapshotId()); - purgeDao.deleteSnapshots(PurgeSnapshotQuery.create().setRootSnapshotId(snapshot.getSnapshotId()), session, profiler); - purgeDao.deleteSnapshots(PurgeSnapshotQuery.create().setId(snapshot.getSnapshotId()), session, profiler); + purgeDao.deleteSnapshots( + session, profiler, + PurgeSnapshotQuery.create().setRootSnapshotId(snapshot.getSnapshotId()), + PurgeSnapshotQuery.create().setId(snapshot.getSnapshotId())); } } diff --git a/sonar-db/src/test/java/org/sonar/db/purge/period/DefaultPeriodCleanerTest.java b/sonar-db/src/test/java/org/sonar/db/purge/period/DefaultPeriodCleanerTest.java index 9a6add0ca06..785504ba073 100644 --- a/sonar-db/src/test/java/org/sonar/db/purge/period/DefaultPeriodCleanerTest.java +++ b/sonar-db/src/test/java/org/sonar/db/purge/period/DefaultPeriodCleanerTest.java @@ -58,8 +58,7 @@ public class DefaultPeriodCleanerTest { verify(filter1).log(); verify(filter2).log(); - verify(dao, times(2)).deleteSnapshots(argThat(newRootSnapshotQuery()), eq(session), any(PurgeProfiler.class)); - verify(dao, times(2)).deleteSnapshots(argThat(newSnapshotIdQuery()), eq(session), any(PurgeProfiler.class)); + verify(dao, times(2)).deleteSnapshots(eq(session), any(PurgeProfiler.class), argThat(newRootSnapshotQuery()), argThat(newSnapshotIdQuery())); } private BaseMatcher newRootSnapshotQuery() {