diff options
Diffstat (limited to 'server')
6 files changed, 115 insertions, 13 deletions
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java index 04a81ef1208..cc3d4c49a2d 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java @@ -26,11 +26,13 @@ import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.utils.System2; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.ce.CeActivityDto; +import static java.util.stream.Stream.concat; +import static org.sonar.core.util.stream.MoreCollectors.toSet; + @ComputeEngineSide public class PurgeCeActivities implements Startable { @@ -48,16 +50,26 @@ public class PurgeCeActivities implements Startable { public void start() { try (DbSession dbSession = dbClient.openSession(false)) { Calendar sixMonthsAgo = Calendar.getInstance(); - sixMonthsAgo.setTimeInMillis(system2.now()); + long now = system2.now(); + sixMonthsAgo.setTimeInMillis(now); sixMonthsAgo.add(Calendar.DATE, -180); LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime()); Set<String> ceActivityUuids = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTimeInMillis()) .stream() .map(CeActivityDto::getUuid) - .collect(MoreCollectors.toSet()); + .collect(toSet()); dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids); - dbClient.ceScannerContextDao().deleteByUuids(dbSession, ceActivityUuids); + + Calendar fourWeeksAgo = Calendar.getInstance(); + fourWeeksAgo.setTimeInMillis(system2.now()); + fourWeeksAgo.add(Calendar.DATE, -28); + + LOGGER.info("Delete the Scanner contexts tasks created before {}", fourWeeksAgo.getTime()); + Set<String> scannerContextUuids = dbClient.ceScannerContextDao().selectOlderThan(dbSession, fourWeeksAgo.getTimeInMillis()); + dbClient.ceScannerContextDao().deleteByUuids( + dbSession, + concat(ceActivityUuids.stream(), scannerContextUuids.stream()).collect(toSet())); dbSession.commit(); } } diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java index 1fd2316eaf5..e492ca9bcff 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java @@ -19,6 +19,9 @@ */ package org.sonar.ce.queue; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Optional; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.System2; @@ -27,13 +30,14 @@ import org.sonar.db.ce.CeActivityDto; import org.sonar.db.ce.CeQueueDto; import org.sonar.db.ce.CeTaskTypes; +import static java.time.ZoneOffset.UTC; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class PurgeCeActivitiesTest { - private System2 system2 = spy(System2.INSTANCE); + private System2 system2 = mock(System2.class); @Rule public DbTester dbTester = DbTester.create(system2); @@ -41,18 +45,53 @@ public class PurgeCeActivitiesTest { private PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2); @Test - public void delete_older_than_6_months() throws Exception { - insertWithDate("VERY_OLD", 1_000_000_000_000L); - insertWithDate("RECENT", 1_500_000_000_000L); - when(system2.now()).thenReturn(1_500_000_000_100L); + public void delete_activity_older_than_180_days_and_their_scanner_context() { + LocalDateTime now = LocalDateTime.now(); + insertWithDate("VERY_OLD", now.minusDays(180).minusMonths(10)); + insertWithDate("JUST_OLD_ENOUGH", now.minusDays(180).minusDays(1)); + insertWithDate("NOT_OLD_ENOUGH", now.minusDays(180)); + insertWithDate("RECENT", now.minusDays(1)); + when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli()); underTest.start(); - assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "VERY_OLD").isPresent()).isFalse(); - assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "RECENT").isPresent()).isTrue(); + assertThat(selectActivity("VERY_OLD").isPresent()).isFalse(); + assertThat(scannerContextExists("VERY_OLD")).isFalse(); + assertThat(selectActivity("JUST_OLD_ENOUGH").isPresent()).isFalse(); + assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse(); + assertThat(selectActivity("NOT_OLD_ENOUGH").isPresent()).isTrue(); + assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isFalse(); // because more than 4 weeks old + assertThat(selectActivity("RECENT").isPresent()).isTrue(); + assertThat(scannerContextExists("RECENT")).isTrue(); } - private void insertWithDate(String uuid, long date) { + @Test + public void delete_ce_scanner_context_older_than_28_days() { + LocalDateTime now = LocalDateTime.now(); + insertWithDate("VERY_OLD", now.minusDays(28).minusMonths(12)); + insertWithDate("JUST_OLD_ENOUGH", now.minusDays(28).minusDays(1)); + insertWithDate("NOT_OLD_ENOUGH", now.minusDays(28)); + insertWithDate("RECENT", now.minusDays(1)); + when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli()); + + underTest.start(); + + assertThat(scannerContextExists("VERY_OLD")).isFalse(); + assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse(); + assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isTrue(); + assertThat(scannerContextExists("RECENT")).isTrue(); + } + + private Optional<CeActivityDto> selectActivity(String very_old) { + return dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), very_old); + } + + private boolean scannerContextExists(String uuid) { + return dbTester.countSql("select count(1) from ce_scanner_context where task_uuid = '" + uuid + "'") == 1; + } + + private void insertWithDate(String uuid, LocalDateTime dateTime) { + long date = dateTime.toInstant(UTC).toEpochMilli(); CeQueueDto queueDto = new CeQueueDto(); queueDto.setUuid(uuid); queueDto.setTaskType(CeTaskTypes.REPORT); @@ -62,5 +101,17 @@ public class PurgeCeActivitiesTest { when(system2.now()).thenReturn(date); dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto); dbTester.getSession().commit(); + + insertScannerContext(uuid, date); + } + + private void insertScannerContext(String uuid, long createdAt) { + dbTester.executeInsert( + "CE_SCANNER_CONTEXT", + "task_uuid", uuid, + "created_at", createdAt, + "updated_at", 1, + "context_data", "YoloContent".getBytes()); + dbTester.commit(); } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextDao.java index 2bdf88965a4..ba3c1e2b04f 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextDao.java @@ -28,6 +28,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; import java.util.Optional; +import java.util.Set; import org.apache.commons.io.IOUtils; import org.sonar.api.utils.System2; import org.sonar.core.util.CloseableIterator; @@ -86,6 +87,10 @@ public class CeScannerContextDao implements Dao { } } + public Set<String> selectOlderThan(DbSession dbSession, long beforeDate) { + return mapper(dbSession).selectOlderThan(beforeDate); + } + public void deleteByUuids(DbSession dbSession, Collection<String> uuids) { DatabaseUtils.executeLargeUpdates(uuids, mapper(dbSession)::deleteByUuids); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java index 9d116d17a28..b2ecfb98650 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java @@ -20,9 +20,12 @@ package org.sonar.db.ce; import java.util.List; +import java.util.Set; import org.apache.ibatis.annotations.Param; public interface CeScannerContextMapper { void deleteByUuids(@Param("uuids") List<String> uuids); + + Set<String> selectOlderThan(@Param("beforeDate") long beforeDate); } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml index a89db6814e8..3068a0fbb40 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml @@ -8,4 +8,13 @@ where task_uuid in <foreach collection="uuids" open="(" close=")" item="uuid" separator=",">#{uuid,jdbcType=VARCHAR}</foreach> </delete> + <select id="selectOlderThan" parameterType="long" resultType="String"> + select + task_uuid + from ce_scanner_context csc + where + csc.created_at < #{beforeDate,jdbcType=BIGINT} + </select> + + </mapper> diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java index be6a8305dbc..8540a705c0a 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java @@ -122,6 +122,28 @@ public class CeScannerContextDaoTest { assertThat(underTest.selectScannerContext(dbSession, "UUID_3")).isEmpty(); } + @Test + public void selectOlderThan() { + insertWithCreationDate("TASK_1", 1_450_000_000_000L); + insertWithCreationDate("TASK_2", 1_460_000_000_000L); + insertWithCreationDate("TASK_3", 1_470_000_000_000L); + + assertThat(underTest.selectOlderThan(dbSession, 1_465_000_000_000L)) + .containsOnly("TASK_1", "TASK_2"); + assertThat(underTest.selectOlderThan(dbSession, 1_450_000_000_000L)) + .isEmpty(); + } + + private void insertWithCreationDate(String uuid, long createdAt) { + dbTester.executeInsert( + "CE_SCANNER_CONTEXT", + "task_uuid", uuid, + "created_at", createdAt, + "updated_at", 1, + "context_data", "YoloContent".getBytes()); + dbSession.commit(); + } + private String insertScannerContext(String uuid) { String data = "data of " + uuid; underTest.insert(dbSession, uuid, scannerContextInputStreamOf(data)); |