SONAR-10416 Purge child tables of CE Queue at startup

6 files changed, 92 insertions, 13 deletions

diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java
index aec3f85c76f..097d184bb9e 100644
--- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java
@@ -58,6 +58,8 @@ public class PurgeCeActivities implements Startable {
         .map(CeActivityDto::getUuid)
         .collect(toSet());
       dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids);
+      dbClient.ceTaskCharacteristicsDao().deleteByTaskUuids(dbSession, ceActivityUuids);
+      dbClient.ceTaskInputDao().deleteByUuids(dbSession, ceActivityUuids);
 
       Date fourWeeksAgo = DateUtils.addDays(new Date(system2.now()), -28);
 
diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java
index 2e5fef83265..fb8bdbfa3de 100644
--- a/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java
+++ b/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java
@@ -19,18 +19,26 @@
  */
 package org.sonar.ce.queue;
 
+import java.nio.charset.Charset;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
+import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
+import org.apache.commons.io.IOUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.utils.System2;
+import org.sonar.core.util.UuidFactoryFast;
 import org.sonar.db.DbTester;
 import org.sonar.db.ce.CeActivityDto;
 import org.sonar.db.ce.CeQueueDto;
+import org.sonar.db.ce.CeTaskCharacteristicDto;
+import org.sonar.db.ce.CeTaskInputDao;
 import org.sonar.db.ce.CeTaskTypes;
 
 import static java.time.ZoneOffset.UTC;
+import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -56,13 +64,25 @@ public class PurgeCeActivitiesTest {
     underTest.start();
 
     assertThat(selectActivity("VERY_OLD").isPresent()).isFalse();
+    assertThat(selectTaskInput("VERY_OLD").isPresent()).isFalse();
+    assertThat(selectTaskCharecteristic("VERY_OLD")).hasSize(0);
     assertThat(scannerContextExists("VERY_OLD")).isFalse();
+
     assertThat(selectActivity("JUST_OLD_ENOUGH").isPresent()).isFalse();
+    assertThat(selectTaskInput("JUST_OLD_ENOUGH").isPresent()).isFalse();
+    assertThat(selectTaskCharecteristic("JUST_OLD_ENOUGH")).hasSize(0);
     assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();
+
     assertThat(selectActivity("NOT_OLD_ENOUGH").isPresent()).isTrue();
+    assertThat(selectTaskInput("NOT_OLD_ENOUGH").isPresent()).isTrue();
+    assertThat(selectTaskCharecteristic("NOT_OLD_ENOUGH")).hasSize(1);
     assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isFalse(); // because more than 4 weeks old
+
     assertThat(selectActivity("RECENT").isPresent()).isTrue();
+    assertThat(selectTaskInput("RECENT").isPresent()).isTrue();
+    assertThat(selectTaskCharecteristic("RECENT")).hasSize(1);
     assertThat(scannerContextExists("RECENT")).isTrue();
+
   }
 
   @Test
@@ -82,8 +102,16 @@ public class PurgeCeActivitiesTest {
     assertThat(scannerContextExists("RECENT")).isTrue();
   }
 
-  private Optional<CeActivityDto> selectActivity(String very_old) {
-    return dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), very_old);
+  private Optional<CeActivityDto> selectActivity(String taskUuid) {
+    return dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), taskUuid);
+  }
+
+  private List<CeTaskCharacteristicDto> selectTaskCharecteristic(String taskUuid) {
+    return dbTester.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(dbTester.getSession(), Collections.singletonList(taskUuid));
+  }
+
+  private Optional<CeTaskInputDao.DataStream> selectTaskInput(String taskUuid) {
+    return dbTester.getDbClient().ceTaskInputDao().selectData(dbTester.getSession(), taskUuid);
   }
 
   private boolean scannerContextExists(String uuid) {
@@ -99,7 +127,15 @@ public class PurgeCeActivitiesTest {
     CeActivityDto dto = new CeActivityDto(queueDto);
     dto.setStatus(CeActivityDto.Status.SUCCESS);
     when(system2.now()).thenReturn(date);
+    CeTaskCharacteristicDto ceTaskCharacteristicDto = new CeTaskCharacteristicDto()
+      .setUuid(UuidFactoryFast.getInstance().create())
+      .setValue(randomAlphanumeric(10))
+      .setKey(randomAlphanumeric(10))
+      .setTaskUuid(dto.getUuid());
+
+    dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), dto.getUuid(), IOUtils.toInputStream(randomAlphanumeric(10), Charset.forName("UTF-8")));
     dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto);
+    dbTester.getDbClient().ceTaskCharacteristicsDao().insert(dbTester.getSession(), Collections.singletonList(ceTaskCharacteristicDto));
     dbTester.getSession().commit();
 
     insertScannerContext(uuid, date);
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java
index 0eb7ca5583c..1a9f3547cd6 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java
@@ -21,10 +21,12 @@ package org.sonar.db.ce;
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 import org.sonar.db.Dao;
 import org.sonar.db.DbSession;
 
 import static org.sonar.db.DatabaseUtils.executeLargeInputs;
+import static org.sonar.db.DatabaseUtils.executeLargeUpdates;
 
 public class CeTaskCharacteristicDao implements Dao {
 
@@ -38,6 +40,10 @@ public class CeTaskCharacteristicDao implements Dao {
     return executeLargeInputs(taskUuids, uuid -> mapper(dbSession).selectByTaskUuids(uuid));
   }
 
+  public void deleteByTaskUuids(DbSession dbSession, Set<String> taskUuids) {
+    executeLargeUpdates(taskUuids, mapper(dbSession)::deleteByTaskUuids);
+  }
+
   private static CeTaskCharacteristicMapper mapper(DbSession session) {
     return session.getMapper(CeTaskCharacteristicMapper.class);
   }
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicMapper.java
index 01fb720fd71..c4f0c6cabab 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicMapper.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicMapper.java
@@ -28,4 +28,5 @@ public interface CeTaskCharacteristicMapper {
 
   void insert(CeTaskCharacteristicDto taskCharacteristic);
 
+  void deleteByTaskUuids(@Param("taskUuids") List<String> taskUuids);
 }
diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeTaskCharacteristicMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeTaskCharacteristicMapper.xml
index 942b031c50b..95fdfba037d 100644
--- a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeTaskCharacteristicMapper.xml
+++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeTaskCharacteristicMapper.xml
@@ -34,4 +34,13 @@
     )
   </insert>
 
+  <delete id="deleteByTaskUuids" parameterType="string">
+    DELETE
+    FROM ce_task_characteristics
+    WHERE
+      task_uuid IN
+      <foreach collection="taskUuids" open="(" close=")" item="taskUuid" separator=",">
+        #{taskUuid,jdbcType=VARCHAR}
+      </foreach>
+  </delete>
 </mapper>
diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskCharacteristicDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskCharacteristicDaoTest.java
index a49bdc84526..fb926328c7f 100644
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskCharacteristicDaoTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskCharacteristicDaoTest.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.db.ce;
 
+import com.google.common.collect.ImmutableSet;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -26,6 +27,7 @@ import org.sonar.api.utils.System2;
 import org.sonar.db.DbTester;
 
 import static java.util.Arrays.asList;
+import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.tuple;
@@ -40,17 +42,9 @@ public class CeTaskCharacteristicDaoTest {
 
   @Test
   public void selectByTaskUuids() {
-    CeTaskCharacteristicDto dto1 = new CeTaskCharacteristicDto()
-      .setKey("key1")
-      .setValue("value1")
-      .setUuid("uuid1")
-      .setTaskUuid("task1");
-    CeTaskCharacteristicDto dto2 = new CeTaskCharacteristicDto()
-      .setKey("key2")
-      .setValue("value2")
-      .setUuid("uuid2")
-      .setTaskUuid("task2");
-    underTest.insert(dbTester.getSession(), asList(dto1, dto2));
+    insert("key1", "value1", "uuid1", "task1");
+    insert("key2", "value2", "uuid2", "task2");
+
     dbTester.getSession().commit();
 
     assertThat(underTest.selectByTaskUuids(dbTester.getSession(), asList("task1", "task2")))
@@ -61,4 +55,35 @@ public class CeTaskCharacteristicDaoTest {
     assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("unknown"))).isEmpty();
   }
 
+
+  @Test
+  public void deleteByTaskUuids() {
+    insert("key1", "value1", "uuid1", "task1");
+    insert("key2", "value2", "uuid2", "task2");
+    insert("key3", "value3", "uuid3", "task3");
+
+    underTest.deleteByTaskUuids(dbTester.getSession(), ImmutableSet.of("task1", "task3"));
+    assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("task1"))).hasSize(0);
+    assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("task2"))).hasSize(1);
+    assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("task3"))).hasSize(0);
+  }
+
+  @Test
+  public void deleteByTaskUuids_does_nothing_if_uuid_does_not_exist() {
+    insert("key1", "value1", "uuid1", "task1");
+
+    // must not fail
+    underTest.deleteByTaskUuids(dbTester.getSession(), singleton("task2"));
+
+    assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("task1"))).hasSize(1);
+  }
+
+  private void insert(String key, String value, String uuid, String taskUuid) {
+    CeTaskCharacteristicDto dto1 = new CeTaskCharacteristicDto()
+      .setKey(key)
+      .setValue(value)
+      .setUuid(uuid)
+      .setTaskUuid(taskUuid);
+    underTest.insert(dbTester.getSession(), singleton(dto1));
+  }
 }