SONAR-9218 purge ce_scanner_context older than 28 days at startup

diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java
index 8c36b8ee23e1667277aeb0df15128bd5dd05ea37..a7ee2c2e66ff410fec8b99027d42ab9b7704564c 100644 (file)
--- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java
@@ -26,11 +26,13 @@ import org.sonar.api.ce.ComputeEngineSide;
 import org.sonar.api.utils.System2;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
-import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.ce.CeActivityDto;
 
+import static java.util.stream.Stream.concat;
+import static org.sonar.core.util.stream.MoreCollectors.toSet;
+
 @ComputeEngineSide
 public class PurgeCeActivities implements Startable {
 
@@ -48,16 +50,26 @@ public class PurgeCeActivities implements Startable {
   public void start() {
     try (DbSession dbSession = dbClient.openSession(false)) {
       Calendar sixMonthsAgo = Calendar.getInstance();
-      sixMonthsAgo.setTimeInMillis(system2.now());
+      long now = system2.now();
+      sixMonthsAgo.setTimeInMillis(now);
       sixMonthsAgo.add(Calendar.DATE, -180);
 
       LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime());
       Set<String> ceActivityUuids = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTimeInMillis())
         .stream()
         .map(CeActivityDto::getUuid)
-        .collect(MoreCollectors.toSet());
+        .collect(toSet());
       dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids);
-      dbClient.ceScannerContextDao().deleteByUuids(dbSession, ceActivityUuids);
+
+      Calendar fourWeeksAgo = Calendar.getInstance();
+      fourWeeksAgo.setTimeInMillis(system2.now());
+      fourWeeksAgo.add(Calendar.DATE, -28);
+
+      LOGGER.info("Delete the Scanner contexts tasks created before {}", fourWeeksAgo.getTime());
+      Set<String> scannerContextUuids = dbClient.ceScannerContextDao().selectOlderThan(dbSession, fourWeeksAgo.getTimeInMillis());
+      dbClient.ceScannerContextDao().deleteByUuids(
+        dbSession,
+        concat(ceActivityUuids.stream(), scannerContextUuids.stream()).collect(toSet()));
       dbSession.commit();
     }
   }

diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java
index 5cb0603c8058bc84e203eea7983a40fe0cdf706a..2e5fef83265ab765e07f8c1d179450447654f75d 100644 (file)
--- a/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java
+++ b/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java
@@ -19,6 +19,9 @@
  */
 package org.sonar.ce.queue;
 
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.Optional;
 import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.utils.System2;
@@ -27,13 +30,14 @@ import org.sonar.db.ce.CeActivityDto;
 import org.sonar.db.ce.CeQueueDto;
 import org.sonar.db.ce.CeTaskTypes;
 
+import static java.time.ZoneOffset.UTC;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class PurgeCeActivitiesTest {
 
-  private System2 system2 = spy(System2.INSTANCE);
+  private System2 system2 = mock(System2.class);
 
   @Rule
   public DbTester dbTester = DbTester.create(system2);
@@ -41,18 +45,53 @@ public class PurgeCeActivitiesTest {
   private PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2);
 
   @Test
-  public void delete_older_than_6_months() {
-    insertWithDate("VERY_OLD", 1_000_000_000_000L);
-    insertWithDate("RECENT", 1_500_000_000_000L);
-    when(system2.now()).thenReturn(1_500_000_000_100L);
+  public void delete_activity_older_than_180_days_and_their_scanner_context() {
+    LocalDateTime now = LocalDateTime.now();
+    insertWithDate("VERY_OLD", now.minusDays(180).minusMonths(10));
+    insertWithDate("JUST_OLD_ENOUGH", now.minusDays(180).minusDays(1));
+    insertWithDate("NOT_OLD_ENOUGH", now.minusDays(180));
+    insertWithDate("RECENT", now.minusDays(1));
+    when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
 
     underTest.start();
 
-    assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "VERY_OLD").isPresent()).isFalse();
-    assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "RECENT").isPresent()).isTrue();
+    assertThat(selectActivity("VERY_OLD").isPresent()).isFalse();
+    assertThat(scannerContextExists("VERY_OLD")).isFalse();
+    assertThat(selectActivity("JUST_OLD_ENOUGH").isPresent()).isFalse();
+    assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();
+    assertThat(selectActivity("NOT_OLD_ENOUGH").isPresent()).isTrue();
+    assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isFalse(); // because more than 4 weeks old
+    assertThat(selectActivity("RECENT").isPresent()).isTrue();
+    assertThat(scannerContextExists("RECENT")).isTrue();
   }
 
-  private void insertWithDate(String uuid, long date) {
+  @Test
+  public void delete_ce_scanner_context_older_than_28_days() {
+    LocalDateTime now = LocalDateTime.now();
+    insertWithDate("VERY_OLD", now.minusDays(28).minusMonths(12));
+    insertWithDate("JUST_OLD_ENOUGH", now.minusDays(28).minusDays(1));
+    insertWithDate("NOT_OLD_ENOUGH", now.minusDays(28));
+    insertWithDate("RECENT", now.minusDays(1));
+    when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
+
+    underTest.start();
+
+    assertThat(scannerContextExists("VERY_OLD")).isFalse();
+    assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();
+    assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isTrue();
+    assertThat(scannerContextExists("RECENT")).isTrue();
+  }
+
+  private Optional<CeActivityDto> selectActivity(String very_old) {
+    return dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), very_old);
+  }
+
+  private boolean scannerContextExists(String uuid) {
+    return dbTester.countSql("select count(1) from ce_scanner_context where task_uuid = '" + uuid + "'") == 1;
+  }
+
+  private void insertWithDate(String uuid, LocalDateTime dateTime) {
+    long date = dateTime.toInstant(UTC).toEpochMilli();
     CeQueueDto queueDto = new CeQueueDto();
     queueDto.setUuid(uuid);
     queueDto.setTaskType(CeTaskTypes.REPORT);
@@ -62,5 +101,17 @@ public class PurgeCeActivitiesTest {
     when(system2.now()).thenReturn(date);
     dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto);
     dbTester.getSession().commit();
+
+    insertScannerContext(uuid, date);
+  }
+
+  private void insertScannerContext(String uuid, long createdAt) {
+    dbTester.executeInsert(
+      "CE_SCANNER_CONTEXT",
+      "task_uuid", uuid,
+      "created_at", createdAt,
+      "updated_at", 1,
+      "context_data", "YoloContent".getBytes());
+    dbTester.commit();
   }
 }

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextDao.java
index 356e19395882aab3bbf7220ce4d9379a9d7eb0ad..591fcedd389454ca589d9e7a699bbb0b88c7011b 100644 (file)
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextDao.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextDao.java
@@ -28,6 +28,7 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Collection;
 import java.util.Optional;
+import java.util.Set;
 import org.apache.commons.io.IOUtils;
 import org.sonar.api.utils.System2;
 import org.sonar.core.util.CloseableIterator;
@@ -86,6 +87,10 @@ public class CeScannerContextDao implements Dao {
     }
   }
 
+  public Set<String> selectOlderThan(DbSession dbSession, long beforeDate) {
+    return mapper(dbSession).selectOlderThan(beforeDate);
+  }
+
   public void deleteByUuids(DbSession dbSession, Collection<String> uuids) {
     DatabaseUtils.executeLargeUpdates(uuids, mapper(dbSession)::deleteByUuids);
   }

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java
index d7577e291a0af77f6946a7bcc19a9fbb447aa231..c36cd5347713b1ca898b62a0bbe4e63c98d9c2f9 100644 (file)
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java
@@ -20,9 +20,12 @@
 package org.sonar.db.ce;
 
 import java.util.List;
+import java.util.Set;
 import org.apache.ibatis.annotations.Param;
 
 public interface CeScannerContextMapper {
 
   void deleteByUuids(@Param("uuids") List<String> uuids);
+
+  Set<String> selectOlderThan(@Param("beforeDate") long beforeDate);
 }

diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml
index a89db6814e8cce5fea891b19b76ef470de878b1b..3068a0fbb405b390a15425f0e4c49be88ab8afcf 100644 (file)
--- a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml
+++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml
@@ -8,4 +8,13 @@
     where task_uuid in <foreach collection="uuids" open="(" close=")" item="uuid" separator=",">#{uuid,jdbcType=VARCHAR}</foreach>
   </delete>
 
+  <select id="selectOlderThan" parameterType="long" resultType="String">
+    select
+      task_uuid
+    from ce_scanner_context csc
+    where
+      csc.created_at &lt; #{beforeDate,jdbcType=BIGINT}
+  </select>
+
+
 </mapper>

diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java
index 420ec28994fb543f405d1fe835627d99e8aa8fcb..11a489f9aab9b27fd66901b8dad625ab9d51b9f0 100644 (file)
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java
@@ -122,6 +122,28 @@ public class CeScannerContextDaoTest {
     assertThat(underTest.selectScannerContext(dbSession, "UUID_3")).isEmpty();
   }
 
+  @Test
+  public void selectOlderThan() {
+    insertWithCreationDate("TASK_1", 1_450_000_000_000L);
+    insertWithCreationDate("TASK_2", 1_460_000_000_000L);
+    insertWithCreationDate("TASK_3", 1_470_000_000_000L);
+
+    assertThat(underTest.selectOlderThan(dbSession, 1_465_000_000_000L))
+      .containsOnly("TASK_1", "TASK_2");
+    assertThat(underTest.selectOlderThan(dbSession, 1_450_000_000_000L))
+      .isEmpty();
+  }
+
+  private void insertWithCreationDate(String uuid, long createdAt) {
+    dbTester.executeInsert(
+      "CE_SCANNER_CONTEXT",
+      "task_uuid", uuid,
+      "created_at", createdAt,
+      "updated_at", 1,
+      "context_data", "YoloContent".getBytes());
+    dbSession.commit();
+  }
+
   private String insertScannerContext(String uuid) {
     String data = "data of " + uuid;
     underTest.insert(dbSession, uuid, scannerContextInputStreamOf(data));