@@ -26,11 +26,13 @@ import org.sonar.api.ce.ComputeEngineSide; | |||
import org.sonar.api.utils.System2; | |||
import org.sonar.api.utils.log.Logger; | |||
import org.sonar.api.utils.log.Loggers; | |||
import org.sonar.core.util.stream.MoreCollectors; | |||
import org.sonar.db.DbClient; | |||
import org.sonar.db.DbSession; | |||
import org.sonar.db.ce.CeActivityDto; | |||
import static java.util.stream.Stream.concat; | |||
import static org.sonar.core.util.stream.MoreCollectors.toSet; | |||
@ComputeEngineSide | |||
public class PurgeCeActivities implements Startable { | |||
@@ -48,16 +50,26 @@ public class PurgeCeActivities implements Startable { | |||
public void start() { | |||
try (DbSession dbSession = dbClient.openSession(false)) { | |||
Calendar sixMonthsAgo = Calendar.getInstance(); | |||
sixMonthsAgo.setTimeInMillis(system2.now()); | |||
long now = system2.now(); | |||
sixMonthsAgo.setTimeInMillis(now); | |||
sixMonthsAgo.add(Calendar.DATE, -180); | |||
LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime()); | |||
Set<String> ceActivityUuids = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTimeInMillis()) | |||
.stream() | |||
.map(CeActivityDto::getUuid) | |||
.collect(MoreCollectors.toSet()); | |||
.collect(toSet()); | |||
dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids); | |||
dbClient.ceScannerContextDao().deleteByUuids(dbSession, ceActivityUuids); | |||
Calendar fourWeeksAgo = Calendar.getInstance(); | |||
fourWeeksAgo.setTimeInMillis(system2.now()); | |||
fourWeeksAgo.add(Calendar.DATE, -28); | |||
LOGGER.info("Delete the Scanner contexts tasks created before {}", fourWeeksAgo.getTime()); | |||
Set<String> scannerContextUuids = dbClient.ceScannerContextDao().selectOlderThan(dbSession, fourWeeksAgo.getTimeInMillis()); | |||
dbClient.ceScannerContextDao().deleteByUuids( | |||
dbSession, | |||
concat(ceActivityUuids.stream(), scannerContextUuids.stream()).collect(toSet())); | |||
dbSession.commit(); | |||
} | |||
} |
@@ -19,6 +19,9 @@ | |||
*/ | |||
package org.sonar.ce.queue; | |||
import java.time.LocalDateTime; | |||
import java.time.ZoneOffset; | |||
import java.util.Optional; | |||
import org.junit.Rule; | |||
import org.junit.Test; | |||
import org.sonar.api.utils.System2; | |||
@@ -27,13 +30,14 @@ import org.sonar.db.ce.CeActivityDto; | |||
import org.sonar.db.ce.CeQueueDto; | |||
import org.sonar.db.ce.CeTaskTypes; | |||
import static java.time.ZoneOffset.UTC; | |||
import static org.assertj.core.api.Assertions.assertThat; | |||
import static org.mockito.Mockito.spy; | |||
import static org.mockito.Mockito.mock; | |||
import static org.mockito.Mockito.when; | |||
public class PurgeCeActivitiesTest { | |||
private System2 system2 = spy(System2.INSTANCE); | |||
private System2 system2 = mock(System2.class); | |||
@Rule | |||
public DbTester dbTester = DbTester.create(system2); | |||
@@ -41,18 +45,53 @@ public class PurgeCeActivitiesTest { | |||
private PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2); | |||
@Test | |||
public void delete_older_than_6_months() { | |||
insertWithDate("VERY_OLD", 1_000_000_000_000L); | |||
insertWithDate("RECENT", 1_500_000_000_000L); | |||
when(system2.now()).thenReturn(1_500_000_000_100L); | |||
public void delete_activity_older_than_180_days_and_their_scanner_context() { | |||
LocalDateTime now = LocalDateTime.now(); | |||
insertWithDate("VERY_OLD", now.minusDays(180).minusMonths(10)); | |||
insertWithDate("JUST_OLD_ENOUGH", now.minusDays(180).minusDays(1)); | |||
insertWithDate("NOT_OLD_ENOUGH", now.minusDays(180)); | |||
insertWithDate("RECENT", now.minusDays(1)); | |||
when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli()); | |||
underTest.start(); | |||
assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "VERY_OLD").isPresent()).isFalse(); | |||
assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "RECENT").isPresent()).isTrue(); | |||
assertThat(selectActivity("VERY_OLD").isPresent()).isFalse(); | |||
assertThat(scannerContextExists("VERY_OLD")).isFalse(); | |||
assertThat(selectActivity("JUST_OLD_ENOUGH").isPresent()).isFalse(); | |||
assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse(); | |||
assertThat(selectActivity("NOT_OLD_ENOUGH").isPresent()).isTrue(); | |||
assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isFalse(); // because more than 4 weeks old | |||
assertThat(selectActivity("RECENT").isPresent()).isTrue(); | |||
assertThat(scannerContextExists("RECENT")).isTrue(); | |||
} | |||
private void insertWithDate(String uuid, long date) { | |||
@Test | |||
public void delete_ce_scanner_context_older_than_28_days() { | |||
LocalDateTime now = LocalDateTime.now(); | |||
insertWithDate("VERY_OLD", now.minusDays(28).minusMonths(12)); | |||
insertWithDate("JUST_OLD_ENOUGH", now.minusDays(28).minusDays(1)); | |||
insertWithDate("NOT_OLD_ENOUGH", now.minusDays(28)); | |||
insertWithDate("RECENT", now.minusDays(1)); | |||
when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli()); | |||
underTest.start(); | |||
assertThat(scannerContextExists("VERY_OLD")).isFalse(); | |||
assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse(); | |||
assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isTrue(); | |||
assertThat(scannerContextExists("RECENT")).isTrue(); | |||
} | |||
private Optional<CeActivityDto> selectActivity(String very_old) { | |||
return dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), very_old); | |||
} | |||
private boolean scannerContextExists(String uuid) { | |||
return dbTester.countSql("select count(1) from ce_scanner_context where task_uuid = '" + uuid + "'") == 1; | |||
} | |||
private void insertWithDate(String uuid, LocalDateTime dateTime) { | |||
long date = dateTime.toInstant(UTC).toEpochMilli(); | |||
CeQueueDto queueDto = new CeQueueDto(); | |||
queueDto.setUuid(uuid); | |||
queueDto.setTaskType(CeTaskTypes.REPORT); | |||
@@ -62,5 +101,17 @@ public class PurgeCeActivitiesTest { | |||
when(system2.now()).thenReturn(date); | |||
dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto); | |||
dbTester.getSession().commit(); | |||
insertScannerContext(uuid, date); | |||
} | |||
private void insertScannerContext(String uuid, long createdAt) { | |||
dbTester.executeInsert( | |||
"CE_SCANNER_CONTEXT", | |||
"task_uuid", uuid, | |||
"created_at", createdAt, | |||
"updated_at", 1, | |||
"context_data", "YoloContent".getBytes()); | |||
dbTester.commit(); | |||
} | |||
} |
@@ -28,6 +28,7 @@ import java.sql.ResultSet; | |||
import java.sql.SQLException; | |||
import java.util.Collection; | |||
import java.util.Optional; | |||
import java.util.Set; | |||
import org.apache.commons.io.IOUtils; | |||
import org.sonar.api.utils.System2; | |||
import org.sonar.core.util.CloseableIterator; | |||
@@ -86,6 +87,10 @@ public class CeScannerContextDao implements Dao { | |||
} | |||
} | |||
public Set<String> selectOlderThan(DbSession dbSession, long beforeDate) { | |||
return mapper(dbSession).selectOlderThan(beforeDate); | |||
} | |||
public void deleteByUuids(DbSession dbSession, Collection<String> uuids) { | |||
DatabaseUtils.executeLargeUpdates(uuids, mapper(dbSession)::deleteByUuids); | |||
} |
@@ -20,9 +20,12 @@ | |||
package org.sonar.db.ce; | |||
import java.util.List; | |||
import java.util.Set; | |||
import org.apache.ibatis.annotations.Param; | |||
public interface CeScannerContextMapper { | |||
void deleteByUuids(@Param("uuids") List<String> uuids); | |||
Set<String> selectOlderThan(@Param("beforeDate") long beforeDate); | |||
} |
@@ -8,4 +8,13 @@ | |||
where task_uuid in <foreach collection="uuids" open="(" close=")" item="uuid" separator=",">#{uuid,jdbcType=VARCHAR}</foreach> | |||
</delete> | |||
<select id="selectOlderThan" parameterType="long" resultType="String"> | |||
select | |||
task_uuid | |||
from ce_scanner_context csc | |||
where | |||
csc.created_at < #{beforeDate,jdbcType=BIGINT} | |||
</select> | |||
</mapper> |
@@ -122,6 +122,28 @@ public class CeScannerContextDaoTest { | |||
assertThat(underTest.selectScannerContext(dbSession, "UUID_3")).isEmpty(); | |||
} | |||
@Test | |||
public void selectOlderThan() { | |||
insertWithCreationDate("TASK_1", 1_450_000_000_000L); | |||
insertWithCreationDate("TASK_2", 1_460_000_000_000L); | |||
insertWithCreationDate("TASK_3", 1_470_000_000_000L); | |||
assertThat(underTest.selectOlderThan(dbSession, 1_465_000_000_000L)) | |||
.containsOnly("TASK_1", "TASK_2"); | |||
assertThat(underTest.selectOlderThan(dbSession, 1_450_000_000_000L)) | |||
.isEmpty(); | |||
} | |||
private void insertWithCreationDate(String uuid, long createdAt) { | |||
dbTester.executeInsert( | |||
"CE_SCANNER_CONTEXT", | |||
"task_uuid", uuid, | |||
"created_at", createdAt, | |||
"updated_at", 1, | |||
"context_data", "YoloContent".getBytes()); | |||
dbSession.commit(); | |||
} | |||
private String insertScannerContext(String uuid) { | |||
String data = "data of " + uuid; | |||
underTest.insert(dbSession, uuid, scannerContextInputStreamOf(data)); |