*/
package org.sonar.ce.queue;
-import java.util.Date;
-import java.util.Set;
import org.sonar.api.Startable;
import org.sonar.api.ce.ComputeEngineSide;
-import org.sonar.api.utils.DateUtils;
-import org.sonar.api.utils.System2;
-import org.sonar.api.utils.log.Logger;
-import org.sonar.api.utils.log.Loggers;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
-import org.sonar.db.ce.CeActivityDto;
-
-import static java.util.stream.Stream.concat;
-import static org.sonar.core.util.stream.MoreCollectors.toSet;
+import org.sonar.db.purge.PurgeProfiler;
@ComputeEngineSide
public class PurgeCeActivities implements Startable {
- private static final Logger LOGGER = Loggers.get(PurgeCeActivities.class);
-
private final DbClient dbClient;
- private final System2 system2;
+ private final PurgeProfiler profiler;
- public PurgeCeActivities(DbClient dbClient, System2 system2) {
+ public PurgeCeActivities(DbClient dbClient, PurgeProfiler profiler) {
this.dbClient = dbClient;
- this.system2 = system2;
+ this.profiler = profiler;
}
@Override
public void start() {
try (DbSession dbSession = dbClient.openSession(false)) {
- Date sixMonthsAgo = DateUtils.addDays(new Date(system2.now()), -180);
-
- LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime());
- Set<String> ceActivityUuids = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTime())
- .stream()
- .map(CeActivityDto::getUuid)
- .collect(toSet());
- dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids);
- dbClient.ceTaskCharacteristicsDao().deleteByTaskUuids(dbSession, ceActivityUuids);
- dbClient.ceTaskInputDao().deleteByUuids(dbSession, ceActivityUuids);
-
- Date fourWeeksAgo = DateUtils.addDays(new Date(system2.now()), -28);
-
- LOGGER.info("Delete the Scanner contexts tasks created before {}", fourWeeksAgo.getTime());
- Set<String> scannerContextUuids = dbClient.ceScannerContextDao().selectOlderThan(dbSession, fourWeeksAgo.getTime());
- dbClient.ceScannerContextDao().deleteByUuids(
- dbSession,
- concat(ceActivityUuids.stream(), scannerContextUuids.stream()).collect(toSet()));
+ dbClient.purgeDao().purgeCeActivities(dbSession, profiler);
+ dbClient.purgeDao().purgeCeScannerContexts(dbSession, profiler);
dbSession.commit();
}
}
*/
package org.sonar.ce.queue;
-import java.nio.charset.Charset;
-import java.time.LocalDateTime;
-import java.time.ZoneOffset;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import org.apache.commons.io.IOUtils;
-import org.junit.Rule;
import org.junit.Test;
-import org.sonar.api.utils.System2;
-import org.sonar.core.util.UuidFactoryFast;
-import org.sonar.db.DbTester;
-import org.sonar.db.ce.CeActivityDto;
-import org.sonar.db.ce.CeQueueDto;
-import org.sonar.db.ce.CeTaskCharacteristicDto;
-import org.sonar.db.ce.CeTaskInputDao;
-import org.sonar.db.ce.CeTaskTypes;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.purge.PurgeDao;
+import org.sonar.db.purge.PurgeProfiler;
-import static java.time.ZoneOffset.UTC;
-import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
-import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class PurgeCeActivitiesTest {
- private System2 system2 = mock(System2.class);
-
- @Rule
- public DbTester dbTester = DbTester.create(system2);
-
- private PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2);
-
- @Test
- public void delete_activity_older_than_180_days_and_their_scanner_context() {
- LocalDateTime now = LocalDateTime.now();
- insertWithDate("VERY_OLD", now.minusDays(180).minusMonths(10));
- insertWithDate("JUST_OLD_ENOUGH", now.minusDays(180).minusDays(1));
- insertWithDate("NOT_OLD_ENOUGH", now.minusDays(180));
- insertWithDate("RECENT", now.minusDays(1));
- when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
-
- underTest.start();
-
- assertThat(selectActivity("VERY_OLD").isPresent()).isFalse();
- assertThat(selectTaskInput("VERY_OLD").isPresent()).isFalse();
- assertThat(selectTaskCharecteristic("VERY_OLD")).hasSize(0);
- assertThat(scannerContextExists("VERY_OLD")).isFalse();
-
- assertThat(selectActivity("JUST_OLD_ENOUGH").isPresent()).isFalse();
- assertThat(selectTaskInput("JUST_OLD_ENOUGH").isPresent()).isFalse();
- assertThat(selectTaskCharecteristic("JUST_OLD_ENOUGH")).hasSize(0);
- assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();
-
- assertThat(selectActivity("NOT_OLD_ENOUGH").isPresent()).isTrue();
- assertThat(selectTaskInput("NOT_OLD_ENOUGH").isPresent()).isTrue();
- assertThat(selectTaskCharecteristic("NOT_OLD_ENOUGH")).hasSize(1);
- assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isFalse(); // because more than 4 weeks old
-
- assertThat(selectActivity("RECENT").isPresent()).isTrue();
- assertThat(selectTaskInput("RECENT").isPresent()).isTrue();
- assertThat(selectTaskCharecteristic("RECENT")).hasSize(1);
- assertThat(scannerContextExists("RECENT")).isTrue();
-
- }
+ private DbClient dbClient = mock(DbClient.class);
+ private PurgeDao purgeDao = mock(PurgeDao.class);
+ private DbSession dbSession = mock(DbSession.class);
+ private PurgeProfiler profiler = mock(PurgeProfiler.class);
+ private PurgeCeActivities underTest = new PurgeCeActivities(dbClient, profiler);
@Test
- public void delete_ce_scanner_context_older_than_28_days() {
- LocalDateTime now = LocalDateTime.now();
- insertWithDate("VERY_OLD", now.minusDays(28).minusMonths(12));
- insertWithDate("JUST_OLD_ENOUGH", now.minusDays(28).minusDays(1));
- insertWithDate("NOT_OLD_ENOUGH", now.minusDays(28));
- insertWithDate("RECENT", now.minusDays(1));
- when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
+ public void starts_calls_purgeDao_and_commit() {
+ when(dbClient.purgeDao()).thenReturn(purgeDao);
+ when(dbClient.openSession(false)).thenReturn(dbSession);
underTest.start();
- assertThat(scannerContextExists("VERY_OLD")).isFalse();
- assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();
- assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isTrue();
- assertThat(scannerContextExists("RECENT")).isTrue();
- }
-
- private Optional<CeActivityDto> selectActivity(String taskUuid) {
- return dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), taskUuid);
- }
-
- private List<CeTaskCharacteristicDto> selectTaskCharecteristic(String taskUuid) {
- return dbTester.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(dbTester.getSession(), Collections.singletonList(taskUuid));
- }
-
- private Optional<CeTaskInputDao.DataStream> selectTaskInput(String taskUuid) {
- return dbTester.getDbClient().ceTaskInputDao().selectData(dbTester.getSession(), taskUuid);
- }
-
- private boolean scannerContextExists(String uuid) {
- return dbTester.countSql("select count(1) from ce_scanner_context where task_uuid = '" + uuid + "'") == 1;
- }
-
- private void insertWithDate(String uuid, LocalDateTime dateTime) {
- long date = dateTime.toInstant(UTC).toEpochMilli();
- CeQueueDto queueDto = new CeQueueDto();
- queueDto.setUuid(uuid);
- queueDto.setTaskType(CeTaskTypes.REPORT);
-
- CeActivityDto dto = new CeActivityDto(queueDto);
- dto.setStatus(CeActivityDto.Status.SUCCESS);
- when(system2.now()).thenReturn(date);
- CeTaskCharacteristicDto ceTaskCharacteristicDto = new CeTaskCharacteristicDto()
- .setUuid(UuidFactoryFast.getInstance().create())
- .setValue(randomAlphanumeric(10))
- .setKey(randomAlphanumeric(10))
- .setTaskUuid(dto.getUuid());
-
- dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), dto.getUuid(), IOUtils.toInputStream(randomAlphanumeric(10), Charset.forName("UTF-8")));
- dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto);
- dbTester.getDbClient().ceTaskCharacteristicsDao().insert(dbTester.getSession(), Collections.singletonList(ceTaskCharacteristicDto));
- dbTester.getSession().commit();
-
- insertScannerContext(uuid, date);
- }
-
- private void insertScannerContext(String uuid, long createdAt) {
- dbTester.executeInsert(
- "CE_SCANNER_CONTEXT",
- "task_uuid", uuid,
- "created_at", createdAt,
- "updated_at", 1,
- "context_data", "YoloContent".getBytes());
- dbTester.commit();
+ InOrder inOrder = Mockito.inOrder(purgeDao, dbSession);
+ inOrder.verify(purgeDao).purgeCeActivities(dbSession, profiler);
+ inOrder.verify(purgeDao).purgeCeScannerContexts(dbSession, profiler);
+ inOrder.verify(dbSession).commit();
+ inOrder.verify(dbSession).close();
+ inOrder.verifyNoMoreInteractions();
}
}
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
import org.sonar.db.DbSession;
class PurgeCommands {
void deleteCeActivity(String rootUuid) {
profiler.start("deleteCeActivity (ce_scanner_context)");
- purgeMapper.deleteCeScannerContextOfCeActivityByRootUuid(rootUuid);
+ purgeMapper.deleteCeScannerContextOfCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
profiler.start("deleteCeActivity (ce_task_characteristics)");
- purgeMapper.deleteCeTaskCharacteristicsOfCeActivityByRootUuid(rootUuid);
+ purgeMapper.deleteCeTaskCharacteristicsOfCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
profiler.start("deleteCeActivity (ce_task_input)");
- purgeMapper.deleteCeTaskInputOfCeActivityByRootUuid(rootUuid);
+ purgeMapper.deleteCeTaskInputOfCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
profiler.start("deleteCeActivity (ce_task_message)");
- purgeMapper.deleteCeTaskMessageOfCeActivityByRootUuid(rootUuid);
+ purgeMapper.deleteCeTaskMessageOfCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
profiler.start("deleteCeActivity (ce_activity)");
- purgeMapper.deleteCeActivityByRootUuid(rootUuid);
+ purgeMapper.deleteCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
}
+ void deleteCeActivityBefore(@Nullable String rootUuid, long createdAt) {
+ profiler.start("deleteCeActivityBefore (ce_scanner_context)");
+ purgeMapper.deleteCeScannerContextOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
+ session.commit();
+ profiler.stop();
+ profiler.start("deleteCeActivityBefore (ce_task_characteristics)");
+ purgeMapper.deleteCeTaskCharacteristicsOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
+ session.commit();
+ profiler.stop();
+ profiler.start("deleteCeActivityBefore (ce_task_input)");
+ purgeMapper.deleteCeTaskInputOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
+ session.commit();
+ profiler.stop();
+ profiler.start("deleteCeActivityBefore (ce_task_message)");
+ purgeMapper.deleteCeTaskMessageOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
+ session.commit();
+ profiler.stop();
+ profiler.start("deleteCeActivityBefore (ce_activity)");
+ purgeMapper.deleteCeActivityByRootUuidOrBefore(rootUuid, createdAt);
+ session.commit();
+ profiler.stop();
+ }
+
+ void deleteCeScannerContextBefore(@Nullable String rootUuid, long createdAt) {
+ // assuming CeScannerContext of rows in table CE_QUEUE can't be older than createdAt
+ profiler.start("deleteCeScannerContextBefore");
+ purgeMapper.deleteCeScannerContextOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
+ session.commit();
+ }
+
void deleteCeQueue(String rootUuid) {
profiler.start("deleteCeQueue (ce_scanner_context)");
purgeMapper.deleteCeScannerContextOfCeQueueByRootUuid(rootUuid);
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Stream;
+import javax.annotation.Nullable;
+import org.sonar.api.utils.DateUtils;
import org.sonar.api.utils.System2;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
purgeAnalyses(commands, rootUuid);
purgeDisabledComponents(session, mapper, conf, listener);
deleteOldClosedIssues(conf, mapper, listener);
+ purgeOldCeActivities(rootUuid, commands);
+ purgeOldCeScannerContexts(rootUuid, commands);
deleteOldDisabledComponents(commands, mapper, rootUuid);
purgeStaleBranches(commands, conf, mapper, rootUuid);
}
- private static void purgeStaleBranches(PurgeCommands commands, PurgeConfiguration conf, PurgeMapper mapper, String rootUuid) {
+ private void purgeStaleBranches(PurgeCommands commands, PurgeConfiguration conf, PurgeMapper mapper, String rootUuid) {
Optional<Date> maxDate = conf.maxLiveDateOfInactiveShortLivingBranches();
if (!maxDate.isPresent()) {
// not available if branch plugin is not installed
.collect(MoreCollectors.toList());
}
+ public void purgeCeActivities(DbSession session, PurgeProfiler profiler) {
+ PurgeMapper mapper = session.getMapper(PurgeMapper.class);
+ PurgeCommands commands = new PurgeCommands(session, mapper, profiler);
+ purgeOldCeActivities(null, commands);
+ }
+
+ private void purgeOldCeActivities(@Nullable String rootUuid, PurgeCommands commands) {
+ Date sixMonthsAgo = DateUtils.addDays(new Date(system2.now()), -180);
+ commands.deleteCeActivityBefore(rootUuid, sixMonthsAgo.getTime());
+ }
+
+ public void purgeCeScannerContexts(DbSession session, PurgeProfiler profiler) {
+ PurgeMapper mapper = session.getMapper(PurgeMapper.class);
+ PurgeCommands commands = new PurgeCommands(session, mapper, profiler);
+ purgeOldCeScannerContexts(null, commands);
+ }
+
+ private void purgeOldCeScannerContexts(@Nullable String rootUuid, PurgeCommands commands) {
+ Date fourWeeksAgo = DateUtils.addDays(new Date(system2.now()), -28);
+ commands.deleteCeScannerContextBefore(rootUuid, fourWeeksAgo.getTime());
+ }
+
+
private static final class ManualBaselineAnalysisFilter implements Predicate<PurgeableAnalysisDto> {
private static final String[] NO_BASELINE = {null};
deleteRootComponent(uuid, purgeMapper, purgeCommands);
}
- private static void deleteRootComponent(String rootUuid, PurgeMapper mapper, PurgeCommands commands) {
+ private void deleteRootComponent(String rootUuid, PurgeMapper mapper, PurgeCommands commands) {
List<IdUuidPair> rootAndModulesOrSubviews = mapper.selectRootAndModulesOrSubviewsByProjectUuid(rootUuid);
long rootId = rootAndModulesOrSubviews.stream()
.filter(pair -> pair.getUuid().equals(rootUuid))
@CheckForNull
String selectManualBaseline(@Param("projectUuid") String projectUuid);
- List<IdUuidPair> selectDisabledComponentsWithoutIssues(@Param("projectUuid") String projectUuid);
+ List<IdUuidPair> selectDisabledComponentsWithoutIssues(@Param("projectUuid") String projectUuid);
void deleteIssuesFromKeys(@Param("keys") List<String> keys);
void deleteFileSourcesByFileUuid(@Param("fileUuids") List<String> fileUuids);
- void deleteCeTaskCharacteristicsOfCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
+ void deleteCeTaskCharacteristicsOfCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
+ @Nullable @Param("createdAtBefore") Long createdAtBefore);
- void deleteCeTaskInputOfCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
+ void deleteCeTaskInputOfCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
+ @Nullable @Param("createdAtBefore") Long createdAtBefore);
- void deleteCeScannerContextOfCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
+ void deleteCeScannerContextOfCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
+ @Nullable @Param("createdAtBefore") Long createdAtBefore);
- void deleteCeTaskMessageOfCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
+ void deleteCeTaskMessageOfCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
+ @Nullable @Param("createdAtBefore") Long createdAtBefore);
- void deleteCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
+ /**
+ * Delete rows in CE_ACTIVITY of tasks of the specified component and/or created before specified date.
+ */
+ void deleteCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
+ @Nullable @Param("createdAtBefore") Long createdAtBefore);
void deleteCeScannerContextOfCeQueueByRootUuid(@Param("rootUuid") String rootUuid);
</foreach>
</delete>
- <delete id="deleteCeScannerContextOfCeActivityByRootUuid">
+ <delete id="deleteCeScannerContextOfCeActivityByRootUuidOrBefore">
delete from ce_scanner_context
where
- task_uuid in (
- select
- uuid
- from ce_activity
- where
- component_uuid=#{rootUuid,jdbcType=VARCHAR}
- or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
- )
+ task_uuid in (
+ select
+ uuid
+ from ce_activity
+ <include refid="whereClauseCeActivityByRootUuidOrBefore" />
+ )
</delete>
- <delete id="deleteCeTaskCharacteristicsOfCeActivityByRootUuid">
+ <delete id="deleteCeTaskCharacteristicsOfCeActivityByRootUuidOrBefore">
delete from ce_task_characteristics
where
- task_uuid in (
- select
- uuid
- from ce_activity
- where
- component_uuid=#{rootUuid,jdbcType=VARCHAR}
- or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
- )
+ task_uuid in (
+ select
+ uuid
+ from ce_activity
+ <include refid="whereClauseCeActivityByRootUuidOrBefore" />
+ )
</delete>
- <delete id="deleteCeTaskInputOfCeActivityByRootUuid">
+ <delete id="deleteCeTaskInputOfCeActivityByRootUuidOrBefore">
delete from ce_task_input
where
- task_uuid in (
- select
- uuid
- from ce_activity
- where
- component_uuid=#{rootUuid,jdbcType=VARCHAR}
- or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
- )
+ task_uuid in (
+ select
+ uuid
+ from ce_activity
+ <include refid="whereClauseCeActivityByRootUuidOrBefore" />
+ )
</delete>
- <delete id="deleteCeTaskMessageOfCeActivityByRootUuid">
+ <delete id="deleteCeTaskMessageOfCeActivityByRootUuidOrBefore">
delete from ce_task_message
where
- task_uuid in (
- select
- uuid
- from ce_activity
- where
- component_uuid=#{rootUuid,jdbcType=VARCHAR}
- or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
- )
+ task_uuid in (
+ select
+ uuid
+ from ce_activity
+ <include refid="whereClauseCeActivityByRootUuidOrBefore" />
+ )
</delete>
- <delete id="deleteCeActivityByRootUuid">
- delete from ce_activity
- where
+ <delete id="deleteCeActivityByRootUuidOrBefore">
+ delete from ce_activity
+ <include refid="whereClauseCeActivityByRootUuidOrBefore" />
+ </delete>
+
+ <sql id="whereClauseCeActivityByRootUuidOrBefore">
+ where
+ <choose>
+ <when test="rootUuid != null and createdAtBefore != null">
+ created_at < #{createdAtBefore,jdbcType=BIGINT}
+ and (
+ component_uuid=#{rootUuid,jdbcType=VARCHAR}
+ or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
+ )
+ </when>
+ <when test="createdAtBefore != null">
+ created_at < #{createdAtBefore,jdbcType=BIGINT}
+ </when>
+ <when test="rootUuid != null">
component_uuid=#{rootUuid,jdbcType=VARCHAR}
or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
- </delete>
+ </when>
+ <!-- safety net when both variables are null to never generate a
+ delete statement deleting the whole table -->
+ <otherwise>
+ 1 = 2
+ </otherwise>
+ </choose>
+ </sql>
<delete id="deleteCeScannerContextOfCeQueueByRootUuid">
delete from ce_scanner_context
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.io.ByteArrayInputStream;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.Random;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeQueueDto.Status;
import org.sonar.db.ce.CeTaskCharacteristicDto;
+import org.sonar.db.ce.CeTaskInputDao;
import org.sonar.db.ce.CeTaskMessageDto;
+import org.sonar.db.ce.CeTaskTypes;
import org.sonar.db.component.BranchDto;
import org.sonar.db.component.BranchType;
import org.sonar.db.component.ComponentDbTester;
import org.sonar.db.webhook.WebhookDto;
import static com.google.common.base.MoreObjects.firstNonNull;
+import static java.time.ZoneOffset.UTC;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
enabledFileWithoutIssues.uuid());
}
+ @Test
+ public void delete_ce_analysis_older_than_180_and_scanner_context_older_than_40_days_of_specified_project_when_purging_project() {
+ LocalDateTime now = LocalDateTime.now();
+ ComponentDto project1 = db.components().insertPublicProject();
+ Consumer<CeQueueDto> belongsToProject1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(project1.uuid());
+ ComponentDto project2 = db.components().insertPublicProject();
+ Consumer<CeQueueDto> belongsToProject2 = t -> t.setMainComponentUuid(project2.uuid()).setComponentUuid(project2.uuid());
+
+ insertCeActivityAndChildDataWithDate("VERY_OLD_1", now.minusDays(180).minusMonths(10), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_1", now.minusDays(180).minusDays(1), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_1", now.minusDays(180), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("RECENT_1", now.minusDays(1), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("VERY_OLD_2", now.minusDays(180).minusMonths(10), belongsToProject2);
+ insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_2", now.minusDays(180).minusDays(1), belongsToProject2);
+ insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_2", now.minusDays(180), belongsToProject2);
+ insertCeActivityAndChildDataWithDate("RECENT_2", now.minusDays(1), belongsToProject2);
+
+ when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
+ underTest.purge(db.getSession(), newConfigurationWith30Days(System2.INSTANCE, project1.uuid(), project1.uuid()),
+ PurgeListener.EMPTY, new PurgeProfiler());
+
+ assertThat(selectActivity("VERY_OLD_1")).isEmpty();
+ assertThat(selectTaskInput("VERY_OLD_1")).isEmpty();
+ assertThat(selectTaskCharacteristic("VERY_OLD_1")).hasSize(0);
+ assertThat(scannerContextExists("VERY_OLD_1")).isFalse();
+ assertThat(selectActivity("VERY_OLD_2")).isNotEmpty();
+ assertThat(selectTaskInput("VERY_OLD_2")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("VERY_OLD_2")).hasSize(1);
+ assertThat(scannerContextExists("VERY_OLD_2")).isTrue();
+
+ assertThat(selectActivity("JUST_OLD_ENOUGH_1")).isEmpty();
+ assertThat(selectTaskInput("JUST_OLD_ENOUGH_1")).isEmpty();
+ assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_1")).hasSize(0);
+ assertThat(scannerContextExists("JUST_OLD_ENOUGH_1")).isFalse();
+ assertThat(selectActivity("JUST_OLD_ENOUGH_2")).isNotEmpty();
+ assertThat(selectTaskInput("JUST_OLD_ENOUGH_2")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_2")).hasSize(1);
+ assertThat(scannerContextExists("JUST_OLD_ENOUGH_2")).isTrue();
+
+ assertThat(selectActivity("NOT_OLD_ENOUGH_1")).isNotEmpty();
+ assertThat(selectTaskInput("NOT_OLD_ENOUGH_1")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_1")).hasSize(1);
+ assertThat(scannerContextExists("NOT_OLD_ENOUGH_1")).isFalse(); // because more than 4 weeks old
+ assertThat(selectActivity("NOT_OLD_ENOUGH_2")).isNotEmpty();
+ assertThat(selectTaskInput("NOT_OLD_ENOUGH_2")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_2")).hasSize(1);
+ assertThat(scannerContextExists("NOT_OLD_ENOUGH_2")).isTrue();
+
+ assertThat(selectActivity("RECENT_1")).isNotEmpty();
+ assertThat(selectTaskInput("RECENT_1")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("RECENT_1")).hasSize(1);
+ assertThat(scannerContextExists("RECENT_1")).isTrue();
+ assertThat(selectActivity("RECENT_2")).isNotEmpty();
+ assertThat(selectTaskInput("RECENT_2")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("RECENT_2")).hasSize(1);
+ assertThat(scannerContextExists("RECENT_2")).isTrue();
+ }
+
+ @Test
+ public void delete_ce_analysis_older_than_180_and_scanner_context_older_than_40_days_of_project_and_branches_when_purging_project() {
+ LocalDateTime now = LocalDateTime.now();
+ ComponentDto project1 = db.components().insertPublicProject();
+ ComponentDto branch1 = db.components().insertProjectBranch(project1);
+ Consumer<CeQueueDto> belongsToProject1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(project1.uuid());
+ Consumer<CeQueueDto> belongsToBranch1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(branch1.uuid());
+
+ insertCeActivityAndChildDataWithDate("VERY_OLD_1", now.minusDays(180).minusMonths(10), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_1", now.minusDays(180).minusDays(1), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_1", now.minusDays(180), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("RECENT_1", now.minusDays(1), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("VERY_OLD_2", now.minusDays(180).minusMonths(10), belongsToBranch1);
+ insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_2", now.minusDays(180).minusDays(1), belongsToBranch1);
+ insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_2", now.minusDays(180), belongsToBranch1);
+ insertCeActivityAndChildDataWithDate("RECENT_2", now.minusDays(1), belongsToBranch1);
+
+ when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
+ underTest.purge(db.getSession(), newConfigurationWith30Days(System2.INSTANCE, project1.uuid(), project1.uuid()),
+ PurgeListener.EMPTY, new PurgeProfiler());
+
+ assertThat(selectActivity("VERY_OLD_1")).isEmpty();
+ assertThat(selectTaskInput("VERY_OLD_1")).isEmpty();
+ assertThat(selectTaskCharacteristic("VERY_OLD_1")).hasSize(0);
+ assertThat(scannerContextExists("VERY_OLD_1")).isFalse();
+ assertThat(selectActivity("VERY_OLD_2")).isEmpty();
+ assertThat(selectTaskInput("VERY_OLD_2")).isEmpty();
+ assertThat(selectTaskCharacteristic("VERY_OLD_2")).isEmpty();
+ assertThat(scannerContextExists("VERY_OLD_2")).isFalse();
+
+ assertThat(selectActivity("JUST_OLD_ENOUGH_1")).isEmpty();
+ assertThat(selectTaskInput("JUST_OLD_ENOUGH_1")).isEmpty();
+ assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_1")).hasSize(0);
+ assertThat(scannerContextExists("JUST_OLD_ENOUGH_1")).isFalse();
+ assertThat(selectActivity("JUST_OLD_ENOUGH_2")).isEmpty();
+ assertThat(selectTaskInput("JUST_OLD_ENOUGH_2")).isEmpty();
+ assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_2")).isEmpty();
+ assertThat(scannerContextExists("JUST_OLD_ENOUGH_2")).isFalse();
+
+ assertThat(selectActivity("NOT_OLD_ENOUGH_1")).isNotEmpty();
+ assertThat(selectTaskInput("NOT_OLD_ENOUGH_1")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_1")).hasSize(1);
+ assertThat(scannerContextExists("NOT_OLD_ENOUGH_1")).isFalse(); // because more than 4 weeks old
+ assertThat(selectActivity("NOT_OLD_ENOUGH_2")).isNotEmpty();
+ assertThat(selectTaskInput("NOT_OLD_ENOUGH_2")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_2")).hasSize(1);
+ assertThat(scannerContextExists("NOT_OLD_ENOUGH_2")).isFalse(); // because more than 4 weeks old
+
+ assertThat(selectActivity("RECENT_1")).isNotEmpty();
+ assertThat(selectTaskInput("RECENT_1")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("RECENT_1")).hasSize(1);
+ assertThat(scannerContextExists("RECENT_1")).isTrue();
+ assertThat(selectActivity("RECENT_2")).isNotEmpty();
+ assertThat(selectTaskInput("RECENT_2")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("RECENT_2")).hasSize(1);
+ assertThat(scannerContextExists("RECENT_2")).isTrue();
+ }
+
+ @Test
+ public void delete_ce_analysis_of_branch_older_than_180_and_scanner_context_older_than_40_days_when_purging_branch() {
+ LocalDateTime now = LocalDateTime.now();
+ ComponentDto project1 = db.components().insertPublicProject();
+ ComponentDto branch1 = db.components().insertProjectBranch(project1);
+ Consumer<CeQueueDto> belongsToProject1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(project1.uuid());
+ Consumer<CeQueueDto> belongsToBranch1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(branch1.uuid());
+
+ insertCeActivityAndChildDataWithDate("VERY_OLD_1", now.minusDays(180).minusMonths(10), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_1", now.minusDays(180).minusDays(1), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_1", now.minusDays(180), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("RECENT_1", now.minusDays(1), belongsToProject1);
+ insertCeActivityAndChildDataWithDate("VERY_OLD_2", now.minusDays(180).minusMonths(10), belongsToBranch1);
+ insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_2", now.minusDays(180).minusDays(1), belongsToBranch1);
+ insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_2", now.minusDays(180), belongsToBranch1);
+ insertCeActivityAndChildDataWithDate("RECENT_2", now.minusDays(1), belongsToBranch1);
+
+ when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
+ underTest.purge(db.getSession(), newConfigurationWith30Days(System2.INSTANCE, branch1.uuid(), branch1.uuid()),
+ PurgeListener.EMPTY, new PurgeProfiler());
+
+ assertThat(selectActivity("VERY_OLD_1")).isNotEmpty();
+ assertThat(selectTaskInput("VERY_OLD_1")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("VERY_OLD_1")).hasSize(1);
+ assertThat(scannerContextExists("VERY_OLD_1")).isTrue();
+ assertThat(selectActivity("VERY_OLD_2")).isEmpty();
+ assertThat(selectTaskInput("VERY_OLD_2")).isEmpty();
+ assertThat(selectTaskCharacteristic("VERY_OLD_2")).isEmpty();
+ assertThat(scannerContextExists("VERY_OLD_2")).isFalse();
+
+ assertThat(selectActivity("JUST_OLD_ENOUGH_1")).isNotEmpty();
+ assertThat(selectTaskInput("JUST_OLD_ENOUGH_1")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_1")).hasSize(1);
+ assertThat(scannerContextExists("JUST_OLD_ENOUGH_1")).isTrue();
+ assertThat(selectActivity("JUST_OLD_ENOUGH_2")).isEmpty();
+ assertThat(selectTaskInput("JUST_OLD_ENOUGH_2")).isEmpty();
+ assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_2")).isEmpty();
+ assertThat(scannerContextExists("JUST_OLD_ENOUGH_2")).isFalse();
+
+ assertThat(selectActivity("NOT_OLD_ENOUGH_1")).isNotEmpty();
+ assertThat(selectTaskInput("NOT_OLD_ENOUGH_1")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_1")).hasSize(1);
+ assertThat(scannerContextExists("NOT_OLD_ENOUGH_1")).isTrue();
+ assertThat(selectActivity("NOT_OLD_ENOUGH_2")).isNotEmpty();
+ assertThat(selectTaskInput("NOT_OLD_ENOUGH_2")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_2")).hasSize(1);
+ assertThat(scannerContextExists("NOT_OLD_ENOUGH_2")).isFalse(); // because more than 4 weeks old
+
+ assertThat(selectActivity("RECENT_1")).isNotEmpty();
+ assertThat(selectTaskInput("RECENT_1")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("RECENT_1")).hasSize(1);
+ assertThat(scannerContextExists("RECENT_1")).isTrue();
+ assertThat(selectActivity("RECENT_2")).isNotEmpty();
+ assertThat(selectTaskInput("RECENT_2")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("RECENT_2")).hasSize(1);
+ assertThat(scannerContextExists("RECENT_2")).isTrue();
+ }
+
@Test
public void deleteProject_deletes_webhook_deliveries() {
ComponentDto project = db.components().insertPublicProject();
.containsOnly(view.uuid(), pc.uuid());
}
+ @Test
+ public void purgeCeActivities_deletes_activity_older_than_180_days_and_their_scanner_context() {
+ LocalDateTime now = LocalDateTime.now();
+ insertCeActivityAndChildDataWithDate("VERY_OLD", now.minusDays(180).minusMonths(10));
+ insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH", now.minusDays(180).minusDays(1));
+ insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH", now.minusDays(180));
+ insertCeActivityAndChildDataWithDate("RECENT", now.minusDays(1));
+ when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
+
+ underTest.purgeCeActivities(db.getSession(), new PurgeProfiler());
+
+ assertThat(selectActivity("VERY_OLD")).isEmpty();
+ assertThat(selectTaskInput("VERY_OLD")).isEmpty();
+ assertThat(selectTaskCharacteristic("VERY_OLD")).hasSize(0);
+ assertThat(scannerContextExists("VERY_OLD")).isFalse();
+
+ assertThat(selectActivity("JUST_OLD_ENOUGH")).isEmpty();
+ assertThat(selectTaskInput("JUST_OLD_ENOUGH")).isEmpty();
+ assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH")).hasSize(0);
+ assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();
+
+ assertThat(selectActivity("NOT_OLD_ENOUGH")).isNotEmpty();
+ assertThat(selectTaskInput("NOT_OLD_ENOUGH")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH")).hasSize(1);
+ assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isTrue();
+
+ assertThat(selectActivity("RECENT")).isNotEmpty();
+ assertThat(selectTaskInput("RECENT")).isNotEmpty();
+ assertThat(selectTaskCharacteristic("RECENT")).hasSize(1);
+ assertThat(scannerContextExists("RECENT")).isTrue();
+ }
+
+ @Test
+ public void purgeCeScannerContexts_deletes_ce_scanner_context_older_than_28_days() {
+ LocalDateTime now = LocalDateTime.now();
+ insertCeActivityAndChildDataWithDate("VERY_OLD", now.minusDays(28).minusMonths(12));
+ insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH", now.minusDays(28).minusDays(1));
+ insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH", now.minusDays(28));
+ insertCeActivityAndChildDataWithDate("RECENT", now.minusDays(1));
+ when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
+
+ underTest.purgeCeScannerContexts(db.getSession(), new PurgeProfiler());
+
+ assertThat(scannerContextExists("VERY_OLD")).isFalse();
+ assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();
+ assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isTrue();
+ assertThat(scannerContextExists("RECENT")).isTrue();
+ }
+
+ private Optional<CeActivityDto> selectActivity(String taskUuid) {
+ return db.getDbClient().ceActivityDao().selectByUuid(db.getSession(), taskUuid);
+ }
+
+ private List<CeTaskCharacteristicDto> selectTaskCharacteristic(String taskUuid) {
+ return db.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(db.getSession(), Collections.singletonList(taskUuid));
+ }
+
+ private Optional<CeTaskInputDao.DataStream> selectTaskInput(String taskUuid) {
+ return db.getDbClient().ceTaskInputDao().selectData(db.getSession(), taskUuid);
+ }
+
+ private boolean scannerContextExists(String uuid) {
+ return db.countSql("select count(1) from ce_scanner_context where task_uuid = '" + uuid + "'") == 1;
+ }
+
+ @SafeVarargs
+ private final void insertCeActivityAndChildDataWithDate(String ceActivityUuid, LocalDateTime dateTime,
+ Consumer<CeQueueDto>... queueDtoConsumers) {
+ long date = dateTime.toInstant(UTC).toEpochMilli();
+ CeQueueDto queueDto = new CeQueueDto();
+ queueDto.setUuid(ceActivityUuid);
+ queueDto.setTaskType(CeTaskTypes.REPORT);
+ Arrays.stream(queueDtoConsumers).forEach(t -> t.accept(queueDto));
+ CeActivityDto dto = new CeActivityDto(queueDto);
+ dto.setStatus(CeActivityDto.Status.SUCCESS);
+
+ when(system2.now()).thenReturn(date);
+ insertCeTaskInput(dto.getUuid());
+ insertCeTaskCharacteristics(dto.getUuid(), 1);
+ insertCeScannerContext(dto.getUuid());
+ insertCeTaskMessages(dto.getUuid(), 2);
+ db.getDbClient().ceActivityDao().insert(db.getSession(), dto);
+ db.getSession().commit();
+ }
+
private void insertManualMeasureFor(ComponentDto... componentDtos) {
Arrays.stream(componentDtos).forEach(componentDto -> dbClient.customMeasureDao().insert(dbSession, new CustomMeasureDto()
.setComponentUuid(componentDto.uuid())
projects / sonarqube.git / commitdiff