SONAR-7108 Delete Issues from index when purging old issues

Purge from index was already working when property 'sonar.dbcleaner.daysBeforeDeletingClosedIssues' was not equal to 0. Now it's working whatever the value is.

5 files changed, 80 insertions, 61 deletions

diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java
index fb8c66be654..53bd63175ef 100644
--- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java
+++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java
@@ -19,12 +19,14 @@
  */
 package org.sonar.db.purge;
 
+import com.google.common.base.Function;
 import com.google.common.collect.Lists;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import javax.annotation.Nonnull;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.ibatis.session.ResultContext;
 import org.apache.ibatis.session.ResultHandler;
@@ -39,7 +41,9 @@ import org.sonar.db.MyBatis;
 import org.sonar.db.component.ResourceDao;
 import org.sonar.db.component.ResourceDto;
 
+import static java.util.Collections.emptyList;
 import static org.sonar.api.utils.DateUtils.dateToLong;
+import static org.sonar.db.DatabaseUtils.executeLargeInputs;
 
 /**
  * @since 2.14
@@ -81,13 +85,15 @@ public class PurgeDao implements Dao {
     for (ResourceDto project : projects) {
       disableOrphanResources(project, session, mapper, listener);
     }
-    deleteOldClosedIssues(conf, mapper);
+    deleteOldClosedIssues(conf, mapper, listener);
   }
 
-  private static void deleteOldClosedIssues(PurgeConfiguration conf, PurgeMapper mapper) {
+  private static void deleteOldClosedIssues(PurgeConfiguration conf, PurgeMapper mapper, PurgeListener listener) {
     Date toDate = conf.maxLiveDateOfClosedIssues();
-    mapper.deleteOldClosedIssueChanges(conf.rootProjectIdUuid().getUuid(), dateToLong(toDate));
-    mapper.deleteOldClosedIssues(conf.rootProjectIdUuid().getUuid(), dateToLong(toDate));
+    List<String> issueKeys = mapper.selectOldClosedIssueKeys(conf.rootProjectIdUuid().getUuid(), dateToLong(toDate));
+    executeLargeInputs(issueKeys, new DeleteIssueChangesFromIssueKeys(mapper));
+    executeLargeInputs(issueKeys, new DeleteIssuesFromKeys(mapper));
+    listener.onIssuesRemoval(issueKeys);
   }
 
   private static void deleteAbortedBuilds(ResourceDto project, PurgeCommands commands) {
@@ -104,9 +110,8 @@ public class PurgeDao implements Dao {
       PurgeSnapshotQuery.create()
         .setResourceId(project.getId())
         .setIslast(false)
-        .setNotPurged(true)
-      );
-    for (final Long projectSnapshotId : projectSnapshotIds) {
+        .setNotPurged(true));
+    for (Long projectSnapshotId : projectSnapshotIds) {
       LOG.debug("<- Clean snapshot " + projectSnapshotId);
       if (!ArrayUtils.isEmpty(scopesWithoutHistoricalData)) {
         PurgeSnapshotQuery query = PurgeSnapshotQuery.create()
@@ -209,4 +214,32 @@ public class PurgeDao implements Dao {
   private static PurgeMapper mapper(DbSession session) {
     return session.getMapper(PurgeMapper.class);
   }
+
+  private static class DeleteIssueChangesFromIssueKeys implements Function<List<String>, List<Void>> {
+    private final PurgeMapper mapper;
+
+    private DeleteIssueChangesFromIssueKeys(PurgeMapper mapper) {
+      this.mapper = mapper;
+    }
+
+    @Override
+    public List<Void> apply(@Nonnull List<String> input) {
+      mapper.deleteIssueChangesFromIssueKeys(input);
+      return emptyList();
+    }
+  }
+
+  private static class DeleteIssuesFromKeys implements Function<List<String>, List<Void>> {
+    private final PurgeMapper mapper;
+
+    private DeleteIssuesFromKeys(PurgeMapper mapper) {
+      this.mapper = mapper;
+    }
+
+    @Override
+    public List<Void> apply(@Nonnull List<String> input) {
+      mapper.deleteIssuesFromKeys(input);
+      return emptyList();
+    }
+  }
 }
diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeListener.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeListener.java
index 64e6c06f74b..a2224c2d715 100644
--- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeListener.java
+++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeListener.java
@@ -19,6 +19,8 @@
  */
 package org.sonar.db.purge;
 
+import java.util.List;
+
 public interface PurgeListener {
 
   PurgeListener EMPTY = new PurgeListener() {
@@ -26,7 +28,14 @@ public interface PurgeListener {
     public void onComponentDisabling(String uuid) {
       // do nothing
     }
+
+    @Override
+    public void onIssuesRemoval(List<String> issueKeys) {
+      // do nothing
+    }
   };
 
   void onComponentDisabling(String uuid);
+
+  void onIssuesRemoval(List<String> issueKeys);
 }
diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java
index f8c3149c64a..d412f0ec6f8 100644
--- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java
+++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java
@@ -82,9 +82,11 @@ public interface PurgeMapper {
 
   void deleteComponentIssues(@Param("componentUuids") List<String> componentUuids);
 
-  void deleteOldClosedIssueChanges(@Param("projectUuid") String projectUuid, @Nullable @Param("toDate") Long toDate);
+  List<String> selectOldClosedIssueKeys(@Param("projectUuid") String projectUuid, @Nullable @Param("toDate") Long toDate);
 
-  void deleteOldClosedIssues(@Param("projectUuid") String projectUuid, @Nullable @Param("toDate") Long toDate);
+  void deleteIssuesFromKeys(@Param("keys") List<String> keys);
+
+  void deleteIssueChangesFromIssueKeys(@Param("issueKeys") List<String> issueKeys);
 
   void deleteFileSourcesByProjectUuid(String rootProjectUuid);
 
diff --git a/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml
index aee39d1763a..fb05f2ebe2c 100644
--- a/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml
+++ b/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml
@@ -260,69 +260,38 @@
     delete from file_sources where file_uuid=#{fileUuid}
   </delete>
 
-  <delete id="deleteOldClosedIssueChanges" parameterType="map">
-    delete from issue_changes ic
-    where exists (
-    select * from issues i
-    where i.project_uuid=#{projectUuid} and i.kee=ic.issue_key
+  <select id="selectOldClosedIssueKeys" parameterType="map" resultType="String">
+    SELECT kee FROM issues
+    WHERE project_uuid=#{projectUuid}
     <choose>
       <when test="toDate == null">
-        and i.issue_close_date is not null
+        AND issue_close_date IS NOT NULL
       </when>
       <otherwise>
-        and i.issue_close_date &lt; #{toDate}
+        AND issue_close_date &lt; #{toDate}
       </otherwise>
     </choose>
-    )
-  </delete>
+  </select>
 
-  <!-- Mssql -->
-  <delete id="deleteOldClosedIssueChanges" databaseId="mssql" parameterType="map">
-    delete issue_changes from issue_changes
-    inner join issues on issue_changes.issue_key=issues.kee
-    where issues.project_uuid=#{projectUuid}
-    <choose>
-      <when test="toDate == null">
-        and issues.issue_close_date is not null
-      </when>
-      <otherwise>
-        and issues.issue_close_date &lt; #{toDate}
-      </otherwise>
-    </choose>
+  <delete id="deleteIssuesFromKeys" parameterType="map">
+    DELETE FROM issues
+    WHERE kee IN
+    <foreach collection="keys" open="(" close=")" item="key" separator=",">
+      #{key}
+    </foreach>
   </delete>
 
-  <!-- Mysql -->
-  <delete id="deleteOldClosedIssueChanges" databaseId="mysql" parameterType="map">
-    delete ic
-    from issue_changes as ic, issues as i
-    where i.project_uuid=#{projectUuid}
-    and ic.issue_key=i.kee
-    <choose>
-      <when test="toDate == null">
-        and i.issue_close_date is not null
-      </when>
-      <otherwise>
-        and i.issue_close_date &lt; #{toDate}
-      </otherwise>
-    </choose>
+  <delete id="deleteIssueChangesFromIssueKeys" parameterType="map">
+    DELETE FROM issue_changes
+    WHERE issue_key IN
+    <foreach collection="issueKeys" open="(" close=")" item="issueKey" separator=",">
+      #{issueKey}
+    </foreach>
   </delete>
 
-  <delete id="deleteOldClosedIssues" parameterType="map">
-    delete from issues
-    where project_uuid=#{projectUuid}
-    <choose>
-      <when test="toDate == null">
-        and issue_close_date is not null
-      </when>
-      <otherwise>
-        and issue_close_date &lt; #{toDate}
-      </otherwise>
-    </choose>
+  <delete id="deleteCeActivityByProjectUuid">
+      delete from ce_activity where component_uuid=#{rootProjectUuid}
   </delete>
 
-    <delete id="deleteCeActivityByProjectUuid">
-        delete from ce_activity where component_uuid=#{rootProjectUuid}
-    </delete>
-
 </mapper>
 
diff --git a/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java
index e44fa62511e..92dd370ae72 100644
--- a/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java
+++ b/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java
@@ -33,8 +33,10 @@ import org.sonar.db.ce.CeQueueDto;
 import org.sonar.db.component.ComponentDto;
 import org.sonar.db.component.ComponentTesting;
 
+import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.sonar.db.ce.CeTaskTypes.REPORT;
 
@@ -171,9 +173,13 @@ public class PurgeDaoTest {
 
   @Test
   public void should_delete_old_closed_issues() {
+    PurgeListener purgeListener = mock(PurgeListener.class);
     dbTester.prepareDbUnit(getClass(), "should_delete_old_closed_issues.xml");
-    underTest.purge(newConfigurationWith30Days(), PurgeListener.EMPTY, new PurgeProfiler());
+
+    underTest.purge(newConfigurationWith30Days(), purgeListener, new PurgeProfiler());
+
     dbTester.assertDbUnit(getClass(), "should_delete_old_closed_issues-result.xml", "issues", "issue_changes");
+    verify(purgeListener).onIssuesRemoval(asList("ISSUE-1", "ISSUE-2"));
   }
 
   @Test