mirrors
/
sonarqube
mirror of https://github.com/SonarSource/sonarqube.git
Watch 1
Star 0
Fork 0
Code Issues 0 Releases 237 Wiki Activity
Browse Source

SONAR-11675 delete scan contexts and CE activities on project purge

tags/7.8
Sébastien Lesaint 5 years ago
parent
1c430b3667
commit
aefa1e5ffb
7 changed files with 425 additions and 204 deletions
Split View Show Diff Stats
  1. 6
    33
      server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java
  2. 20
    118
      server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java
  3. 36
    5
      server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java
  4. 29
    2
      server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java
  5. 14
    6
      server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java
  6. 55
    40
      server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml
  7. 265
    0
      server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java

+ 6
- 33
server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java View File

@@ -19,55 +19,28 @@
*/
package org.sonar.ce.queue;

import java.util.Date;
import java.util.Set;
import org.sonar.api.Startable;
import org.sonar.api.ce.ComputeEngineSide;
import org.sonar.api.utils.DateUtils;
import org.sonar.api.utils.System2;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.ce.CeActivityDto;

import static java.util.stream.Stream.concat;
import static org.sonar.core.util.stream.MoreCollectors.toSet;
import org.sonar.db.purge.PurgeProfiler;

@ComputeEngineSide
public class PurgeCeActivities implements Startable {

private static final Logger LOGGER = Loggers.get(PurgeCeActivities.class);

private final DbClient dbClient;
private final System2 system2;
private final PurgeProfiler profiler;

public PurgeCeActivities(DbClient dbClient, System2 system2) {
public PurgeCeActivities(DbClient dbClient, PurgeProfiler profiler) {
this.dbClient = dbClient;
this.system2 = system2;
this.profiler = profiler;
}

@Override
public void start() {
try (DbSession dbSession = dbClient.openSession(false)) {
Date sixMonthsAgo = DateUtils.addDays(new Date(system2.now()), -180);

LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime());
Set<String> ceActivityUuids = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTime())
.stream()
.map(CeActivityDto::getUuid)
.collect(toSet());
dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids);
dbClient.ceTaskCharacteristicsDao().deleteByTaskUuids(dbSession, ceActivityUuids);
dbClient.ceTaskInputDao().deleteByUuids(dbSession, ceActivityUuids);

Date fourWeeksAgo = DateUtils.addDays(new Date(system2.now()), -28);

LOGGER.info("Delete the Scanner contexts tasks created before {}", fourWeeksAgo.getTime());
Set<String> scannerContextUuids = dbClient.ceScannerContextDao().selectOlderThan(dbSession, fourWeeksAgo.getTime());
dbClient.ceScannerContextDao().deleteByUuids(
dbSession,
concat(ceActivityUuids.stream(), scannerContextUuids.stream()).collect(toSet()));
dbClient.purgeDao().purgeCeActivities(dbSession, profiler);
dbClient.purgeDao().purgeCeScannerContexts(dbSession, profiler);
dbSession.commit();
}
}

+ 20
- 118
server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java View File

@@ -19,135 +19,37 @@
*/
package org.sonar.ce.queue;

import java.nio.charset.Charset;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.utils.System2;
import org.sonar.core.util.UuidFactoryFast;
import org.sonar.db.DbTester;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeTaskCharacteristicDto;
import org.sonar.db.ce.CeTaskInputDao;
import org.sonar.db.ce.CeTaskTypes;
import org.mockito.InOrder;
import org.mockito.Mockito;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.purge.PurgeDao;
import org.sonar.db.purge.PurgeProfiler;

import static java.time.ZoneOffset.UTC;
import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class PurgeCeActivitiesTest {

private System2 system2 = mock(System2.class);

@Rule
public DbTester dbTester = DbTester.create(system2);

private PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2);

@Test
public void delete_activity_older_than_180_days_and_their_scanner_context() {
LocalDateTime now = LocalDateTime.now();
insertWithDate("VERY_OLD", now.minusDays(180).minusMonths(10));
insertWithDate("JUST_OLD_ENOUGH", now.minusDays(180).minusDays(1));
insertWithDate("NOT_OLD_ENOUGH", now.minusDays(180));
insertWithDate("RECENT", now.minusDays(1));
when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());

underTest.start();

assertThat(selectActivity("VERY_OLD").isPresent()).isFalse();
assertThat(selectTaskInput("VERY_OLD").isPresent()).isFalse();
assertThat(selectTaskCharecteristic("VERY_OLD")).hasSize(0);
assertThat(scannerContextExists("VERY_OLD")).isFalse();

assertThat(selectActivity("JUST_OLD_ENOUGH").isPresent()).isFalse();
assertThat(selectTaskInput("JUST_OLD_ENOUGH").isPresent()).isFalse();
assertThat(selectTaskCharecteristic("JUST_OLD_ENOUGH")).hasSize(0);
assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();

assertThat(selectActivity("NOT_OLD_ENOUGH").isPresent()).isTrue();
assertThat(selectTaskInput("NOT_OLD_ENOUGH").isPresent()).isTrue();
assertThat(selectTaskCharecteristic("NOT_OLD_ENOUGH")).hasSize(1);
assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isFalse(); // because more than 4 weeks old

assertThat(selectActivity("RECENT").isPresent()).isTrue();
assertThat(selectTaskInput("RECENT").isPresent()).isTrue();
assertThat(selectTaskCharecteristic("RECENT")).hasSize(1);
assertThat(scannerContextExists("RECENT")).isTrue();

}
private DbClient dbClient = mock(DbClient.class);
private PurgeDao purgeDao = mock(PurgeDao.class);
private DbSession dbSession = mock(DbSession.class);
private PurgeProfiler profiler = mock(PurgeProfiler.class);
private PurgeCeActivities underTest = new PurgeCeActivities(dbClient, profiler);

@Test
public void delete_ce_scanner_context_older_than_28_days() {
LocalDateTime now = LocalDateTime.now();
insertWithDate("VERY_OLD", now.minusDays(28).minusMonths(12));
insertWithDate("JUST_OLD_ENOUGH", now.minusDays(28).minusDays(1));
insertWithDate("NOT_OLD_ENOUGH", now.minusDays(28));
insertWithDate("RECENT", now.minusDays(1));
when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
public void starts_calls_purgeDao_and_commit() {
when(dbClient.purgeDao()).thenReturn(purgeDao);
when(dbClient.openSession(false)).thenReturn(dbSession);

underTest.start();

assertThat(scannerContextExists("VERY_OLD")).isFalse();
assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();
assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isTrue();
assertThat(scannerContextExists("RECENT")).isTrue();
}

private Optional<CeActivityDto> selectActivity(String taskUuid) {
return dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), taskUuid);
}

private List<CeTaskCharacteristicDto> selectTaskCharecteristic(String taskUuid) {
return dbTester.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(dbTester.getSession(), Collections.singletonList(taskUuid));
}

private Optional<CeTaskInputDao.DataStream> selectTaskInput(String taskUuid) {
return dbTester.getDbClient().ceTaskInputDao().selectData(dbTester.getSession(), taskUuid);
}

private boolean scannerContextExists(String uuid) {
return dbTester.countSql("select count(1) from ce_scanner_context where task_uuid = '" + uuid + "'") == 1;
}

private void insertWithDate(String uuid, LocalDateTime dateTime) {
long date = dateTime.toInstant(UTC).toEpochMilli();
CeQueueDto queueDto = new CeQueueDto();
queueDto.setUuid(uuid);
queueDto.setTaskType(CeTaskTypes.REPORT);

CeActivityDto dto = new CeActivityDto(queueDto);
dto.setStatus(CeActivityDto.Status.SUCCESS);
when(system2.now()).thenReturn(date);
CeTaskCharacteristicDto ceTaskCharacteristicDto = new CeTaskCharacteristicDto()
.setUuid(UuidFactoryFast.getInstance().create())
.setValue(randomAlphanumeric(10))
.setKey(randomAlphanumeric(10))
.setTaskUuid(dto.getUuid());

dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), dto.getUuid(), IOUtils.toInputStream(randomAlphanumeric(10), Charset.forName("UTF-8")));
dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto);
dbTester.getDbClient().ceTaskCharacteristicsDao().insert(dbTester.getSession(), Collections.singletonList(ceTaskCharacteristicDto));
dbTester.getSession().commit();

insertScannerContext(uuid, date);
}

private void insertScannerContext(String uuid, long createdAt) {
dbTester.executeInsert(
"CE_SCANNER_CONTEXT",
"task_uuid", uuid,
"created_at", createdAt,
"updated_at", 1,
"context_data", "YoloContent".getBytes());
dbTester.commit();
InOrder inOrder = Mockito.inOrder(purgeDao, dbSession);
inOrder.verify(purgeDao).purgeCeActivities(dbSession, profiler);
inOrder.verify(purgeDao).purgeCeScannerContexts(dbSession, profiler);
inOrder.verify(dbSession).commit();
inOrder.verify(dbSession).close();
inOrder.verifyNoMoreInteractions();
}
}

+ 36
- 5
server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java View File

@@ -24,6 +24,7 @@ import com.google.common.collect.Lists;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.sonar.db.DbSession;

class PurgeCommands {
@@ -281,27 +282,57 @@ class PurgeCommands {

void deleteCeActivity(String rootUuid) {
profiler.start("deleteCeActivity (ce_scanner_context)");
purgeMapper.deleteCeScannerContextOfCeActivityByRootUuid(rootUuid);
purgeMapper.deleteCeScannerContextOfCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
profiler.start("deleteCeActivity (ce_task_characteristics)");
purgeMapper.deleteCeTaskCharacteristicsOfCeActivityByRootUuid(rootUuid);
purgeMapper.deleteCeTaskCharacteristicsOfCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
profiler.start("deleteCeActivity (ce_task_input)");
purgeMapper.deleteCeTaskInputOfCeActivityByRootUuid(rootUuid);
purgeMapper.deleteCeTaskInputOfCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
profiler.start("deleteCeActivity (ce_task_message)");
purgeMapper.deleteCeTaskMessageOfCeActivityByRootUuid(rootUuid);
purgeMapper.deleteCeTaskMessageOfCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
profiler.start("deleteCeActivity (ce_activity)");
purgeMapper.deleteCeActivityByRootUuid(rootUuid);
purgeMapper.deleteCeActivityByRootUuidOrBefore(rootUuid, null);
session.commit();
profiler.stop();
}

void deleteCeActivityBefore(@Nullable String rootUuid, long createdAt) {
profiler.start("deleteCeActivityBefore (ce_scanner_context)");
purgeMapper.deleteCeScannerContextOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
session.commit();
profiler.stop();
profiler.start("deleteCeActivityBefore (ce_task_characteristics)");
purgeMapper.deleteCeTaskCharacteristicsOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
session.commit();
profiler.stop();
profiler.start("deleteCeActivityBefore (ce_task_input)");
purgeMapper.deleteCeTaskInputOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
session.commit();
profiler.stop();
profiler.start("deleteCeActivityBefore (ce_task_message)");
purgeMapper.deleteCeTaskMessageOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
session.commit();
profiler.stop();
profiler.start("deleteCeActivityBefore (ce_activity)");
purgeMapper.deleteCeActivityByRootUuidOrBefore(rootUuid, createdAt);
session.commit();
profiler.stop();
}

void deleteCeScannerContextBefore(@Nullable String rootUuid, long createdAt) {
// assuming CeScannerContext of rows in table CE_QUEUE can't be older than createdAt
profiler.start("deleteCeScannerContextBefore");
purgeMapper.deleteCeScannerContextOfCeActivityByRootUuidOrBefore(rootUuid, createdAt);
session.commit();
}

void deleteCeQueue(String rootUuid) {
profiler.start("deleteCeQueue (ce_scanner_context)");
purgeMapper.deleteCeScannerContextOfCeQueueByRootUuid(rootUuid);

+ 29
- 2
server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java View File

@@ -28,6 +28,8 @@ import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.sonar.api.utils.DateUtils;
import org.sonar.api.utils.System2;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
@@ -68,12 +70,14 @@ public class PurgeDao implements Dao {
purgeAnalyses(commands, rootUuid);
purgeDisabledComponents(session, mapper, conf, listener);
deleteOldClosedIssues(conf, mapper, listener);
purgeOldCeActivities(rootUuid, commands);
purgeOldCeScannerContexts(rootUuid, commands);

deleteOldDisabledComponents(commands, mapper, rootUuid);
purgeStaleBranches(commands, conf, mapper, rootUuid);
}

private static void purgeStaleBranches(PurgeCommands commands, PurgeConfiguration conf, PurgeMapper mapper, String rootUuid) {
private void purgeStaleBranches(PurgeCommands commands, PurgeConfiguration conf, PurgeMapper mapper, String rootUuid) {
Optional<Date> maxDate = conf.maxLiveDateOfInactiveShortLivingBranches();
if (!maxDate.isPresent()) {
// not available if branch plugin is not installed
@@ -174,6 +178,29 @@ public class PurgeDao implements Dao {
.collect(MoreCollectors.toList());
}

public void purgeCeActivities(DbSession session, PurgeProfiler profiler) {
PurgeMapper mapper = session.getMapper(PurgeMapper.class);
PurgeCommands commands = new PurgeCommands(session, mapper, profiler);
purgeOldCeActivities(null, commands);
}

private void purgeOldCeActivities(@Nullable String rootUuid, PurgeCommands commands) {
Date sixMonthsAgo = DateUtils.addDays(new Date(system2.now()), -180);
commands.deleteCeActivityBefore(rootUuid, sixMonthsAgo.getTime());
}

public void purgeCeScannerContexts(DbSession session, PurgeProfiler profiler) {
PurgeMapper mapper = session.getMapper(PurgeMapper.class);
PurgeCommands commands = new PurgeCommands(session, mapper, profiler);
purgeOldCeScannerContexts(null, commands);
}

private void purgeOldCeScannerContexts(@Nullable String rootUuid, PurgeCommands commands) {
Date fourWeeksAgo = DateUtils.addDays(new Date(system2.now()), -28);
commands.deleteCeScannerContextBefore(rootUuid, fourWeeksAgo.getTime());
}


private static final class ManualBaselineAnalysisFilter implements Predicate<PurgeableAnalysisDto> {
private static final String[] NO_BASELINE = {null};

@@ -218,7 +245,7 @@ public class PurgeDao implements Dao {
deleteRootComponent(uuid, purgeMapper, purgeCommands);
}

private static void deleteRootComponent(String rootUuid, PurgeMapper mapper, PurgeCommands commands) {
private void deleteRootComponent(String rootUuid, PurgeMapper mapper, PurgeCommands commands) {
List<IdUuidPair> rootAndModulesOrSubviews = mapper.selectRootAndModulesOrSubviewsByProjectUuid(rootUuid);
long rootId = rootAndModulesOrSubviews.stream()
.filter(pair -> pair.getUuid().equals(rootUuid))

+ 14
- 6
server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java View File

@@ -94,7 +94,7 @@ public interface PurgeMapper {
@CheckForNull
String selectManualBaseline(@Param("projectUuid") String projectUuid);

List<IdUuidPair> selectDisabledComponentsWithoutIssues(@Param("projectUuid") String projectUuid);
List<IdUuidPair> selectDisabledComponentsWithoutIssues(@Param("projectUuid") String projectUuid);

void deleteIssuesFromKeys(@Param("keys") List<String> keys);

@@ -104,15 +104,23 @@ public interface PurgeMapper {

void deleteFileSourcesByFileUuid(@Param("fileUuids") List<String> fileUuids);

void deleteCeTaskCharacteristicsOfCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
void deleteCeTaskCharacteristicsOfCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
@Nullable @Param("createdAtBefore") Long createdAtBefore);

void deleteCeTaskInputOfCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
void deleteCeTaskInputOfCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
@Nullable @Param("createdAtBefore") Long createdAtBefore);

void deleteCeScannerContextOfCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
void deleteCeScannerContextOfCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
@Nullable @Param("createdAtBefore") Long createdAtBefore);

void deleteCeTaskMessageOfCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
void deleteCeTaskMessageOfCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
@Nullable @Param("createdAtBefore") Long createdAtBefore);

void deleteCeActivityByRootUuid(@Param("rootUuid") String rootUuid);
/**
* Delete rows in CE_ACTIVITY of tasks of the specified component and/or created before specified date.
*/
void deleteCeActivityByRootUuidOrBefore(@Nullable @Param("rootUuid") String rootUuid,
@Nullable @Param("createdAtBefore") Long createdAtBefore);

void deleteCeScannerContextOfCeQueueByRootUuid(@Param("rootUuid") String rootUuid);


+ 55
- 40
server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml View File

@@ -344,64 +344,79 @@
</foreach>
</delete>

<delete id="deleteCeScannerContextOfCeActivityByRootUuid">
<delete id="deleteCeScannerContextOfCeActivityByRootUuidOrBefore">
delete from ce_scanner_context
where
task_uuid in (
select
uuid
from ce_activity
where
component_uuid=#{rootUuid,jdbcType=VARCHAR}
or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
)
task_uuid in (
select
uuid
from ce_activity
<include refid="whereClauseCeActivityByRootUuidOrBefore" />
)
</delete>

<delete id="deleteCeTaskCharacteristicsOfCeActivityByRootUuid">
<delete id="deleteCeTaskCharacteristicsOfCeActivityByRootUuidOrBefore">
delete from ce_task_characteristics
where
task_uuid in (
select
uuid
from ce_activity
where
component_uuid=#{rootUuid,jdbcType=VARCHAR}
or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
)
task_uuid in (
select
uuid
from ce_activity
<include refid="whereClauseCeActivityByRootUuidOrBefore" />
)
</delete>

<delete id="deleteCeTaskInputOfCeActivityByRootUuid">
<delete id="deleteCeTaskInputOfCeActivityByRootUuidOrBefore">
delete from ce_task_input
where
task_uuid in (
select
uuid
from ce_activity
where
component_uuid=#{rootUuid,jdbcType=VARCHAR}
or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
)
task_uuid in (
select
uuid
from ce_activity
<include refid="whereClauseCeActivityByRootUuidOrBefore" />
)
</delete>

<delete id="deleteCeTaskMessageOfCeActivityByRootUuid">
<delete id="deleteCeTaskMessageOfCeActivityByRootUuidOrBefore">
delete from ce_task_message
where
task_uuid in (
select
uuid
from ce_activity
where
component_uuid=#{rootUuid,jdbcType=VARCHAR}
or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
)
task_uuid in (
select
uuid
from ce_activity
<include refid="whereClauseCeActivityByRootUuidOrBefore" />
)
</delete>

<delete id="deleteCeActivityByRootUuid">
delete from ce_activity
where
<delete id="deleteCeActivityByRootUuidOrBefore">
delete from ce_activity
<include refid="whereClauseCeActivityByRootUuidOrBefore" />
</delete>

<sql id="whereClauseCeActivityByRootUuidOrBefore">
where
<choose>
<when test="rootUuid != null and createdAtBefore != null">
created_at &lt; #{createdAtBefore,jdbcType=BIGINT}
and (
component_uuid=#{rootUuid,jdbcType=VARCHAR}
or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
)
</when>
<when test="createdAtBefore != null">
created_at &lt; #{createdAtBefore,jdbcType=BIGINT}
</when>
<when test="rootUuid != null">
component_uuid=#{rootUuid,jdbcType=VARCHAR}
or main_component_uuid=#{rootUuid,jdbcType=VARCHAR}
</delete>
</when>
<!-- safety net when both variables are null to never generate a
delete statement deleting the whole table -->
<otherwise>
1 = 2
</otherwise>
</choose>
</sql>

<delete id="deleteCeScannerContextOfCeQueueByRootUuid">
delete from ce_scanner_context

+ 265
- 0
server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java View File

@@ -22,12 +22,15 @@ package org.sonar.db.purge;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.io.ByteArrayInputStream;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.Random;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
@@ -50,7 +53,9 @@ import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeQueueDto.Status;
import org.sonar.db.ce.CeTaskCharacteristicDto;
import org.sonar.db.ce.CeTaskInputDao;
import org.sonar.db.ce.CeTaskMessageDto;
import org.sonar.db.ce.CeTaskTypes;
import org.sonar.db.component.BranchDto;
import org.sonar.db.component.BranchType;
import org.sonar.db.component.ComponentDbTester;
@@ -73,6 +78,7 @@ import org.sonar.db.webhook.WebhookDeliveryLiteDto;
import org.sonar.db.webhook.WebhookDto;

import static com.google.common.base.MoreObjects.firstNonNull;
import static java.time.ZoneOffset.UTC;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
@@ -1024,6 +1030,180 @@ public class PurgeDaoTest {
enabledFileWithoutIssues.uuid());
}

@Test
public void delete_ce_analysis_older_than_180_and_scanner_context_older_than_40_days_of_specified_project_when_purging_project() {
LocalDateTime now = LocalDateTime.now();
ComponentDto project1 = db.components().insertPublicProject();
Consumer<CeQueueDto> belongsToProject1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(project1.uuid());
ComponentDto project2 = db.components().insertPublicProject();
Consumer<CeQueueDto> belongsToProject2 = t -> t.setMainComponentUuid(project2.uuid()).setComponentUuid(project2.uuid());

insertCeActivityAndChildDataWithDate("VERY_OLD_1", now.minusDays(180).minusMonths(10), belongsToProject1);
insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_1", now.minusDays(180).minusDays(1), belongsToProject1);
insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_1", now.minusDays(180), belongsToProject1);
insertCeActivityAndChildDataWithDate("RECENT_1", now.minusDays(1), belongsToProject1);
insertCeActivityAndChildDataWithDate("VERY_OLD_2", now.minusDays(180).minusMonths(10), belongsToProject2);
insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_2", now.minusDays(180).minusDays(1), belongsToProject2);
insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_2", now.minusDays(180), belongsToProject2);
insertCeActivityAndChildDataWithDate("RECENT_2", now.minusDays(1), belongsToProject2);

when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
underTest.purge(db.getSession(), newConfigurationWith30Days(System2.INSTANCE, project1.uuid(), project1.uuid()),
PurgeListener.EMPTY, new PurgeProfiler());

assertThat(selectActivity("VERY_OLD_1")).isEmpty();
assertThat(selectTaskInput("VERY_OLD_1")).isEmpty();
assertThat(selectTaskCharacteristic("VERY_OLD_1")).hasSize(0);
assertThat(scannerContextExists("VERY_OLD_1")).isFalse();
assertThat(selectActivity("VERY_OLD_2")).isNotEmpty();
assertThat(selectTaskInput("VERY_OLD_2")).isNotEmpty();
assertThat(selectTaskCharacteristic("VERY_OLD_2")).hasSize(1);
assertThat(scannerContextExists("VERY_OLD_2")).isTrue();

assertThat(selectActivity("JUST_OLD_ENOUGH_1")).isEmpty();
assertThat(selectTaskInput("JUST_OLD_ENOUGH_1")).isEmpty();
assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_1")).hasSize(0);
assertThat(scannerContextExists("JUST_OLD_ENOUGH_1")).isFalse();
assertThat(selectActivity("JUST_OLD_ENOUGH_2")).isNotEmpty();
assertThat(selectTaskInput("JUST_OLD_ENOUGH_2")).isNotEmpty();
assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_2")).hasSize(1);
assertThat(scannerContextExists("JUST_OLD_ENOUGH_2")).isTrue();

assertThat(selectActivity("NOT_OLD_ENOUGH_1")).isNotEmpty();
assertThat(selectTaskInput("NOT_OLD_ENOUGH_1")).isNotEmpty();
assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_1")).hasSize(1);
assertThat(scannerContextExists("NOT_OLD_ENOUGH_1")).isFalse(); // because more than 4 weeks old
assertThat(selectActivity("NOT_OLD_ENOUGH_2")).isNotEmpty();
assertThat(selectTaskInput("NOT_OLD_ENOUGH_2")).isNotEmpty();
assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_2")).hasSize(1);
assertThat(scannerContextExists("NOT_OLD_ENOUGH_2")).isTrue();

assertThat(selectActivity("RECENT_1")).isNotEmpty();
assertThat(selectTaskInput("RECENT_1")).isNotEmpty();
assertThat(selectTaskCharacteristic("RECENT_1")).hasSize(1);
assertThat(scannerContextExists("RECENT_1")).isTrue();
assertThat(selectActivity("RECENT_2")).isNotEmpty();
assertThat(selectTaskInput("RECENT_2")).isNotEmpty();
assertThat(selectTaskCharacteristic("RECENT_2")).hasSize(1);
assertThat(scannerContextExists("RECENT_2")).isTrue();
}

@Test
public void delete_ce_analysis_older_than_180_and_scanner_context_older_than_40_days_of_project_and_branches_when_purging_project() {
LocalDateTime now = LocalDateTime.now();
ComponentDto project1 = db.components().insertPublicProject();
ComponentDto branch1 = db.components().insertProjectBranch(project1);
Consumer<CeQueueDto> belongsToProject1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(project1.uuid());
Consumer<CeQueueDto> belongsToBranch1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(branch1.uuid());

insertCeActivityAndChildDataWithDate("VERY_OLD_1", now.minusDays(180).minusMonths(10), belongsToProject1);
insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_1", now.minusDays(180).minusDays(1), belongsToProject1);
insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_1", now.minusDays(180), belongsToProject1);
insertCeActivityAndChildDataWithDate("RECENT_1", now.minusDays(1), belongsToProject1);
insertCeActivityAndChildDataWithDate("VERY_OLD_2", now.minusDays(180).minusMonths(10), belongsToBranch1);
insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_2", now.minusDays(180).minusDays(1), belongsToBranch1);
insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_2", now.minusDays(180), belongsToBranch1);
insertCeActivityAndChildDataWithDate("RECENT_2", now.minusDays(1), belongsToBranch1);

when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
underTest.purge(db.getSession(), newConfigurationWith30Days(System2.INSTANCE, project1.uuid(), project1.uuid()),
PurgeListener.EMPTY, new PurgeProfiler());

assertThat(selectActivity("VERY_OLD_1")).isEmpty();
assertThat(selectTaskInput("VERY_OLD_1")).isEmpty();
assertThat(selectTaskCharacteristic("VERY_OLD_1")).hasSize(0);
assertThat(scannerContextExists("VERY_OLD_1")).isFalse();
assertThat(selectActivity("VERY_OLD_2")).isEmpty();
assertThat(selectTaskInput("VERY_OLD_2")).isEmpty();
assertThat(selectTaskCharacteristic("VERY_OLD_2")).isEmpty();
assertThat(scannerContextExists("VERY_OLD_2")).isFalse();

assertThat(selectActivity("JUST_OLD_ENOUGH_1")).isEmpty();
assertThat(selectTaskInput("JUST_OLD_ENOUGH_1")).isEmpty();
assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_1")).hasSize(0);
assertThat(scannerContextExists("JUST_OLD_ENOUGH_1")).isFalse();
assertThat(selectActivity("JUST_OLD_ENOUGH_2")).isEmpty();
assertThat(selectTaskInput("JUST_OLD_ENOUGH_2")).isEmpty();
assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_2")).isEmpty();
assertThat(scannerContextExists("JUST_OLD_ENOUGH_2")).isFalse();

assertThat(selectActivity("NOT_OLD_ENOUGH_1")).isNotEmpty();
assertThat(selectTaskInput("NOT_OLD_ENOUGH_1")).isNotEmpty();
assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_1")).hasSize(1);
assertThat(scannerContextExists("NOT_OLD_ENOUGH_1")).isFalse(); // because more than 4 weeks old
assertThat(selectActivity("NOT_OLD_ENOUGH_2")).isNotEmpty();
assertThat(selectTaskInput("NOT_OLD_ENOUGH_2")).isNotEmpty();
assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_2")).hasSize(1);
assertThat(scannerContextExists("NOT_OLD_ENOUGH_2")).isFalse(); // because more than 4 weeks old

assertThat(selectActivity("RECENT_1")).isNotEmpty();
assertThat(selectTaskInput("RECENT_1")).isNotEmpty();
assertThat(selectTaskCharacteristic("RECENT_1")).hasSize(1);
assertThat(scannerContextExists("RECENT_1")).isTrue();
assertThat(selectActivity("RECENT_2")).isNotEmpty();
assertThat(selectTaskInput("RECENT_2")).isNotEmpty();
assertThat(selectTaskCharacteristic("RECENT_2")).hasSize(1);
assertThat(scannerContextExists("RECENT_2")).isTrue();
}

@Test
public void delete_ce_analysis_of_branch_older_than_180_and_scanner_context_older_than_40_days_when_purging_branch() {
LocalDateTime now = LocalDateTime.now();
ComponentDto project1 = db.components().insertPublicProject();
ComponentDto branch1 = db.components().insertProjectBranch(project1);
Consumer<CeQueueDto> belongsToProject1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(project1.uuid());
Consumer<CeQueueDto> belongsToBranch1 = t -> t.setMainComponentUuid(project1.uuid()).setComponentUuid(branch1.uuid());

insertCeActivityAndChildDataWithDate("VERY_OLD_1", now.minusDays(180).minusMonths(10), belongsToProject1);
insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_1", now.minusDays(180).minusDays(1), belongsToProject1);
insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_1", now.minusDays(180), belongsToProject1);
insertCeActivityAndChildDataWithDate("RECENT_1", now.minusDays(1), belongsToProject1);
insertCeActivityAndChildDataWithDate("VERY_OLD_2", now.minusDays(180).minusMonths(10), belongsToBranch1);
insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH_2", now.minusDays(180).minusDays(1), belongsToBranch1);
insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH_2", now.minusDays(180), belongsToBranch1);
insertCeActivityAndChildDataWithDate("RECENT_2", now.minusDays(1), belongsToBranch1);

when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());
underTest.purge(db.getSession(), newConfigurationWith30Days(System2.INSTANCE, branch1.uuid(), branch1.uuid()),
PurgeListener.EMPTY, new PurgeProfiler());

assertThat(selectActivity("VERY_OLD_1")).isNotEmpty();
assertThat(selectTaskInput("VERY_OLD_1")).isNotEmpty();
assertThat(selectTaskCharacteristic("VERY_OLD_1")).hasSize(1);
assertThat(scannerContextExists("VERY_OLD_1")).isTrue();
assertThat(selectActivity("VERY_OLD_2")).isEmpty();
assertThat(selectTaskInput("VERY_OLD_2")).isEmpty();
assertThat(selectTaskCharacteristic("VERY_OLD_2")).isEmpty();
assertThat(scannerContextExists("VERY_OLD_2")).isFalse();

assertThat(selectActivity("JUST_OLD_ENOUGH_1")).isNotEmpty();
assertThat(selectTaskInput("JUST_OLD_ENOUGH_1")).isNotEmpty();
assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_1")).hasSize(1);
assertThat(scannerContextExists("JUST_OLD_ENOUGH_1")).isTrue();
assertThat(selectActivity("JUST_OLD_ENOUGH_2")).isEmpty();
assertThat(selectTaskInput("JUST_OLD_ENOUGH_2")).isEmpty();
assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH_2")).isEmpty();
assertThat(scannerContextExists("JUST_OLD_ENOUGH_2")).isFalse();

assertThat(selectActivity("NOT_OLD_ENOUGH_1")).isNotEmpty();
assertThat(selectTaskInput("NOT_OLD_ENOUGH_1")).isNotEmpty();
assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_1")).hasSize(1);
assertThat(scannerContextExists("NOT_OLD_ENOUGH_1")).isTrue();
assertThat(selectActivity("NOT_OLD_ENOUGH_2")).isNotEmpty();
assertThat(selectTaskInput("NOT_OLD_ENOUGH_2")).isNotEmpty();
assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH_2")).hasSize(1);
assertThat(scannerContextExists("NOT_OLD_ENOUGH_2")).isFalse(); // because more than 4 weeks old

assertThat(selectActivity("RECENT_1")).isNotEmpty();
assertThat(selectTaskInput("RECENT_1")).isNotEmpty();
assertThat(selectTaskCharacteristic("RECENT_1")).hasSize(1);
assertThat(scannerContextExists("RECENT_1")).isTrue();
assertThat(selectActivity("RECENT_2")).isNotEmpty();
assertThat(selectTaskInput("RECENT_2")).isNotEmpty();
assertThat(selectTaskCharacteristic("RECENT_2")).hasSize(1);
assertThat(scannerContextExists("RECENT_2")).isTrue();
}

@Test
public void deleteProject_deletes_webhook_deliveries() {
ComponentDto project = db.components().insertPublicProject();
@@ -1263,6 +1443,91 @@ public class PurgeDaoTest {
.containsOnly(view.uuid(), pc.uuid());
}

@Test
public void purgeCeActivities_deletes_activity_older_than_180_days_and_their_scanner_context() {
LocalDateTime now = LocalDateTime.now();
insertCeActivityAndChildDataWithDate("VERY_OLD", now.minusDays(180).minusMonths(10));
insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH", now.minusDays(180).minusDays(1));
insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH", now.minusDays(180));
insertCeActivityAndChildDataWithDate("RECENT", now.minusDays(1));
when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());

underTest.purgeCeActivities(db.getSession(), new PurgeProfiler());

assertThat(selectActivity("VERY_OLD")).isEmpty();
assertThat(selectTaskInput("VERY_OLD")).isEmpty();
assertThat(selectTaskCharacteristic("VERY_OLD")).hasSize(0);
assertThat(scannerContextExists("VERY_OLD")).isFalse();

assertThat(selectActivity("JUST_OLD_ENOUGH")).isEmpty();
assertThat(selectTaskInput("JUST_OLD_ENOUGH")).isEmpty();
assertThat(selectTaskCharacteristic("JUST_OLD_ENOUGH")).hasSize(0);
assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();

assertThat(selectActivity("NOT_OLD_ENOUGH")).isNotEmpty();
assertThat(selectTaskInput("NOT_OLD_ENOUGH")).isNotEmpty();
assertThat(selectTaskCharacteristic("NOT_OLD_ENOUGH")).hasSize(1);
assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isTrue();

assertThat(selectActivity("RECENT")).isNotEmpty();
assertThat(selectTaskInput("RECENT")).isNotEmpty();
assertThat(selectTaskCharacteristic("RECENT")).hasSize(1);
assertThat(scannerContextExists("RECENT")).isTrue();
}

@Test
public void purgeCeScannerContexts_deletes_ce_scanner_context_older_than_28_days() {
LocalDateTime now = LocalDateTime.now();
insertCeActivityAndChildDataWithDate("VERY_OLD", now.minusDays(28).minusMonths(12));
insertCeActivityAndChildDataWithDate("JUST_OLD_ENOUGH", now.minusDays(28).minusDays(1));
insertCeActivityAndChildDataWithDate("NOT_OLD_ENOUGH", now.minusDays(28));
insertCeActivityAndChildDataWithDate("RECENT", now.minusDays(1));
when(system2.now()).thenReturn(now.toInstant(ZoneOffset.UTC).toEpochMilli());

underTest.purgeCeScannerContexts(db.getSession(), new PurgeProfiler());

assertThat(scannerContextExists("VERY_OLD")).isFalse();
assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse();
assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isTrue();
assertThat(scannerContextExists("RECENT")).isTrue();
}

private Optional<CeActivityDto> selectActivity(String taskUuid) {
return db.getDbClient().ceActivityDao().selectByUuid(db.getSession(), taskUuid);
}

private List<CeTaskCharacteristicDto> selectTaskCharacteristic(String taskUuid) {
return db.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(db.getSession(), Collections.singletonList(taskUuid));
}

private Optional<CeTaskInputDao.DataStream> selectTaskInput(String taskUuid) {
return db.getDbClient().ceTaskInputDao().selectData(db.getSession(), taskUuid);
}

private boolean scannerContextExists(String uuid) {
return db.countSql("select count(1) from ce_scanner_context where task_uuid = '" + uuid + "'") == 1;
}

@SafeVarargs
private final void insertCeActivityAndChildDataWithDate(String ceActivityUuid, LocalDateTime dateTime,
Consumer<CeQueueDto>... queueDtoConsumers) {
long date = dateTime.toInstant(UTC).toEpochMilli();
CeQueueDto queueDto = new CeQueueDto();
queueDto.setUuid(ceActivityUuid);
queueDto.setTaskType(CeTaskTypes.REPORT);
Arrays.stream(queueDtoConsumers).forEach(t -> t.accept(queueDto));
CeActivityDto dto = new CeActivityDto(queueDto);
dto.setStatus(CeActivityDto.Status.SUCCESS);

when(system2.now()).thenReturn(date);
insertCeTaskInput(dto.getUuid());
insertCeTaskCharacteristics(dto.getUuid(), 1);
insertCeScannerContext(dto.getUuid());
insertCeTaskMessages(dto.getUuid(), 2);
db.getDbClient().ceActivityDao().insert(db.getSession(), dto);
db.getSession().commit();
}

private void insertManualMeasureFor(ComponentDto... componentDtos) {
Arrays.stream(componentDtos).forEach(componentDto -> dbClient.customMeasureDao().insert(dbSession, new CustomMeasureDto()
.setComponentUuid(componentDto.uuid())

Write Preview
Loading…
Cancel
Save