SONAR-7705 purge: delete measures of FILE and DIR via analysis uuid

4 files changed, 99 insertions, 27 deletions

diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java
index c959a949795..3273834f56a 100644
--- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java
+++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java
@@ -175,10 +175,28 @@ class PurgeCommands {
     profiler.stop();
   }
 
+  public void deleteComponentMeasures(List<String> analysisUuids, List<String> componentUuids) {
+    if (analysisUuids.isEmpty() || componentUuids.isEmpty()) {
+      return;
+    }
+
+    List<List<String>> analysisUuidsPartitions = Lists.partition(analysisUuids, MAX_SNAPSHOTS_PER_QUERY);
+    List<List<String>> componentUuidsPartitions = Lists.partition(componentUuids, MAX_RESOURCES_PER_QUERY);
+
+    profiler.start("deleteComponentMeasures");
+    for (List<String> analysisUuidsPartition : analysisUuidsPartitions) {
+      for (List<String> componentUuidsPartition : componentUuidsPartitions) {
+        purgeMapper.deleteComponentMeasures(analysisUuidsPartition, componentUuidsPartition);
+      }
+    }
+    session.commit();
+    profiler.stop();
+  }
+
   void deleteAnalyses(PurgeSnapshotQuery... queries) {
     List<IdUuidPair> snapshotIds = from(asList(queries))
-        .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids)
-        .toList();
+      .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids)
+      .toList();
     deleteAnalyses(snapshotIds);
   }
 
@@ -209,23 +227,22 @@ class PurgeCommands {
   void purgeSnapshots(PurgeSnapshotQuery... queries) {
     // use LinkedHashSet to keep order by remove duplicated ids
     LinkedHashSet<IdUuidPair> snapshotIds = Sets.newLinkedHashSet(from(asList(queries))
-        .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids));
+      .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids));
     purgeSnapshots(snapshotIds);
   }
 
   @VisibleForTesting
   protected void purgeSnapshots(Iterable<IdUuidPair> snapshotIdUuidPairs) {
     // note that events are not deleted
-    List<List<Long>> snapshotIdsPartition = Lists.partition(IdUuidPairs.ids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY);
-    List<List<String>> snapshotUuidsPartition = Lists.partition(IdUuidPairs.uuids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY);
+    List<List<Long>> snapshotIdsPartitions = Lists.partition(IdUuidPairs.ids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY);
+    List<List<String>> snapshotUuidsPartitions = Lists.partition(IdUuidPairs.uuids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY);
 
-    deleteSnapshotDuplications(snapshotUuidsPartition);
+    deleteSnapshotDuplications(snapshotUuidsPartitions);
 
     profiler.start("deleteSnapshotWastedMeasures (project_measures)");
     List<Long> metricIdsWithoutHistoricalData = purgeMapper.selectMetricIdsWithoutHistoricalData();
-    for (List<Long> partSnapshotIds : snapshotIdsPartition) {
-      purgeMapper.deleteSnapshotWastedMeasures(partSnapshotIds, metricIdsWithoutHistoricalData);
-    }
+    snapshotIdsPartitions.stream()
+        .forEach(snapshotIdsPartition -> purgeMapper.deleteSnapshotWastedMeasures(snapshotIdsPartition, metricIdsWithoutHistoricalData));
     session.commit();
     profiler.stop();
 
@@ -235,9 +252,9 @@ class PurgeCommands {
     profiler.stop();
   }
 
-  private void deleteSnapshotDuplications(List<List<String>> snapshotUuidsPartition) {
+  private void deleteSnapshotDuplications(List<List<String>> snapshotUuidsPartitions) {
     profiler.start("deleteSnapshotDuplications (duplications_index)");
-    snapshotUuidsPartition.forEach(purgeMapper::deleteSnapshotDuplications);
+    snapshotUuidsPartitions.forEach(purgeMapper::deleteSnapshotDuplications);
     session.commit();
     profiler.stop();
   }
diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java
index 96df1ac2db1..79fdd11aed4 100644
--- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java
+++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java
@@ -21,17 +21,21 @@ package org.sonar.db.purge;
 
 import com.google.common.collect.Lists;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
-import org.apache.commons.lang.ArrayUtils;
 import org.apache.ibatis.session.SqlSession;
 import org.sonar.api.resources.Scopes;
 import org.sonar.api.utils.System2;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
+import org.sonar.core.util.stream.GuavaCollectors;
 import org.sonar.db.Dao;
 import org.sonar.db.DbSession;
+import org.sonar.db.component.ComponentDao;
+import org.sonar.db.component.ComponentDto;
+import org.sonar.db.component.ComponentTreeQuery;
 import org.sonar.db.component.ResourceDao;
 import org.sonar.db.component.ResourceDto;
 
@@ -45,12 +49,15 @@ import static org.sonar.db.DatabaseUtils.executeLargeInputs;
 public class PurgeDao implements Dao {
   private static final Logger LOG = Loggers.get(PurgeDao.class);
   private static final String[] UNPROCESSED_STATUS = new String[] {"U"};
+  private static final List<String> UUID_FIELD_SORT = Collections.singletonList("uuid");
 
   private final ResourceDao resourceDao;
+  private final ComponentDao componentDao;
   private final System2 system2;
 
-  public PurgeDao(ResourceDao resourceDao, System2 system2) {
+  public PurgeDao(ResourceDao resourceDao, ComponentDao componentDao, System2 system2) {
     this.resourceDao = resourceDao;
+    this.componentDao = componentDao;
     this.system2 = system2;
   }
 
@@ -58,10 +65,12 @@ public class PurgeDao implements Dao {
     PurgeMapper mapper = session.getMapper(PurgeMapper.class);
     PurgeCommands commands = new PurgeCommands(session, mapper, profiler);
     deleteAbortedAnalyses(conf.rootProjectIdUuid().getUuid(), commands);
+    deleteDataOfComponentsWithoutHistoricalData(session, conf.rootProjectIdUuid().getUuid(), conf.scopesWithoutHistoricalData(), commands);
+    // retrieve all nodes in the tree (including root) with scope=PROJECT
     List<ResourceDto> projects = getProjects(conf.rootProjectIdUuid().getId(), session);
     for (ResourceDto project : projects) {
       LOG.debug("-> Clean " + project.getLongName() + " [id=" + project.getId() + "]");
-      purge(project, conf.scopesWithoutHistoricalData(), commands);
+      purge(project.getUuid(), commands);
     }
     for (ResourceDto project : projects) {
       disableOrphanResources(project, session, mapper, listener);
@@ -92,21 +101,53 @@ public class PurgeDao implements Dao {
     commands.deleteAnalyses(query);
   }
 
-  private static void purge(ResourceDto project, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) {
+  private void deleteDataOfComponentsWithoutHistoricalData(DbSession dbSession, String rootUuid, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) {
+    if (scopesWithoutHistoricalData.length == 0) {
+      return;
+    }
+
+    List<String> analysisUuids = purgeCommands.selectSnapshotUuids(
+      PurgeSnapshotQuery.create()
+        .setComponentUuid(rootUuid)
+        .setIslast(false)
+        .setNotPurged(true));
+    List<String> componentWithoutHistoricalDataUuids = componentDao
+      .selectDescendants(
+        dbSession,
+        newComponentTreeQuery()
+          .setBaseUuid(rootUuid)
+          .setQualifiers(Arrays.asList(scopesWithoutHistoricalData))
+          .build())
+      .stream().map(ComponentDto::uuid)
+      .collect(GuavaCollectors.toList());
+
+    purgeCommands.deleteComponentMeasures(analysisUuids, componentWithoutHistoricalDataUuids);
+    // FIXME remove this when cardinality of snapshots has been changed
+    for (String componentUuid : componentWithoutHistoricalDataUuids) {
+      purgeCommands.deleteSnapshots(PurgeSnapshotQuery.create()
+        .setIslast(false)
+        .setComponentUuid(componentUuid));
+    }
+  }
+
+  /**
+   * Creates a new ComponentTreeQuery.Builder with properties that don't matter here but are mandatory populated.
+   */
+  private static ComponentTreeQuery.Builder newComponentTreeQuery() {
+    return ComponentTreeQuery.builder()
+      .setPage(1)
+      .setPageSize(Integer.MAX_VALUE)
+      .setSortFields(UUID_FIELD_SORT);
+  }
+
+  private static void purge(String componentUuid, PurgeCommands purgeCommands) {
     List<String> projectSnapshotUuids = purgeCommands.selectSnapshotUuids(
-        PurgeSnapshotQuery.create()
-            .setComponentUuid(project.getUuid())
-            .setIslast(false)
-            .setNotPurged(true));
+      PurgeSnapshotQuery.create()
+        .setComponentUuid(componentUuid)
+        .setIslast(false)
+        .setNotPurged(true));
     for (String snapshotUuid : projectSnapshotUuids) {
       LOG.debug("<- Clean analysis " + snapshotUuid);
-      if (!ArrayUtils.isEmpty(scopesWithoutHistoricalData)) {
-        PurgeSnapshotQuery query = PurgeSnapshotQuery.create()
-          .setIslast(false)
-          .setScopes(scopesWithoutHistoricalData)
-          .setAnalysisUuid(snapshotUuid);
-        purgeCommands.deleteSnapshots(query);
-      }
 
       // must be executed at the end for reentrance
       purgeCommands.purgeSnapshots(
diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java
index fc2ec93eff3..5ebe7ded301 100644
--- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java
+++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java
@@ -46,6 +46,8 @@ public interface PurgeMapper {
 
   void deleteSnapshotMeasures(@Param("snapshotIds") List<Long> snapshotIds);
 
+  void deleteComponentMeasures(@Param("analysisUuids") List<String> analysisUuids, @Param("componentUuids") List<String> componentUuids);
+
   List<Long> selectMetricIdsWithoutHistoricalData();
 
   void deleteSnapshotWastedMeasures(@Param("snapshotIds") List<Long> snapshotIds, @Param("mids") List<Long> metricIds);
@@ -95,5 +97,4 @@ public interface PurgeMapper {
   void deleteFileSourcesByUuid(String fileUuid);
 
   void deleteCeActivityByProjectUuid(String projectUuid);
-
 }
diff --git a/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml
index a1bcc631507..b087ab53da1 100644
--- a/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml
+++ b/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml
@@ -109,6 +109,19 @@
       </foreach>
   </delete>
 
+  <delete id="deleteComponentMeasures" parameterType="map">
+    delete from project_measures
+    where
+      analysis_uuid in
+      <foreach collection="analysisUuids" open="(" close=")" item="analysisUuid" separator=",">
+        #{analysisUuid}
+      </foreach>
+      and component_uuid in
+      <foreach collection="componentUuids" open="(" close=")" item="componentUuid" separator=",">
+        #{componentUuid}
+      </foreach>
+  </delete>
+
   <delete id="deleteSnapshotDuplications" parameterType="map">
     delete from duplications_index where analysis_uuid in
     <foreach collection="analysisUuids" open="(" close=")" item="analysisUuid" separator=",">