From 8b9805dad763f38fe2e3e1f92ead6accd6c32531 Mon Sep 17 00:00:00 2001 From: Eric Hartmann Date: Tue, 5 Jun 2018 12:26:16 +0200 Subject: [PATCH] SONAR-10416 Purge child tables of CE Queue at startup --- .../org/sonar/ce/queue/PurgeCeActivities.java | 2 + .../sonar/ce/queue/PurgeCeActivitiesTest.java | 40 +++++++++++++++- .../sonar/db/ce/CeTaskCharacteristicDao.java | 6 +++ .../db/ce/CeTaskCharacteristicMapper.java | 1 + .../db/ce/CeTaskCharacteristicMapper.xml | 9 ++++ .../db/ce/CeTaskCharacteristicDaoTest.java | 47 ++++++++++++++----- 6 files changed, 92 insertions(+), 13 deletions(-) diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java index aec3f85c76f..097d184bb9e 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/queue/PurgeCeActivities.java @@ -58,6 +58,8 @@ public class PurgeCeActivities implements Startable { .map(CeActivityDto::getUuid) .collect(toSet()); dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids); + dbClient.ceTaskCharacteristicsDao().deleteByTaskUuids(dbSession, ceActivityUuids); + dbClient.ceTaskInputDao().deleteByUuids(dbSession, ceActivityUuids); Date fourWeeksAgo = DateUtils.addDays(new Date(system2.now()), -28); diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java index 2e5fef83265..fb8bdbfa3de 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/queue/PurgeCeActivitiesTest.java @@ -19,18 +19,26 @@ */ package org.sonar.ce.queue; +import java.nio.charset.Charset; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.util.Collections; +import java.util.List; import java.util.Optional; +import org.apache.commons.io.IOUtils; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.System2; +import org.sonar.core.util.UuidFactoryFast; import org.sonar.db.DbTester; import org.sonar.db.ce.CeActivityDto; import org.sonar.db.ce.CeQueueDto; +import org.sonar.db.ce.CeTaskCharacteristicDto; +import org.sonar.db.ce.CeTaskInputDao; import org.sonar.db.ce.CeTaskTypes; import static java.time.ZoneOffset.UTC; +import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -56,13 +64,25 @@ public class PurgeCeActivitiesTest { underTest.start(); assertThat(selectActivity("VERY_OLD").isPresent()).isFalse(); + assertThat(selectTaskInput("VERY_OLD").isPresent()).isFalse(); + assertThat(selectTaskCharecteristic("VERY_OLD")).hasSize(0); assertThat(scannerContextExists("VERY_OLD")).isFalse(); + assertThat(selectActivity("JUST_OLD_ENOUGH").isPresent()).isFalse(); + assertThat(selectTaskInput("JUST_OLD_ENOUGH").isPresent()).isFalse(); + assertThat(selectTaskCharecteristic("JUST_OLD_ENOUGH")).hasSize(0); assertThat(scannerContextExists("JUST_OLD_ENOUGH")).isFalse(); + assertThat(selectActivity("NOT_OLD_ENOUGH").isPresent()).isTrue(); + assertThat(selectTaskInput("NOT_OLD_ENOUGH").isPresent()).isTrue(); + assertThat(selectTaskCharecteristic("NOT_OLD_ENOUGH")).hasSize(1); assertThat(scannerContextExists("NOT_OLD_ENOUGH")).isFalse(); // because more than 4 weeks old + assertThat(selectActivity("RECENT").isPresent()).isTrue(); + assertThat(selectTaskInput("RECENT").isPresent()).isTrue(); + assertThat(selectTaskCharecteristic("RECENT")).hasSize(1); assertThat(scannerContextExists("RECENT")).isTrue(); + } @Test @@ -82,8 +102,16 @@ public class PurgeCeActivitiesTest { assertThat(scannerContextExists("RECENT")).isTrue(); } - private Optional selectActivity(String very_old) { - return dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), very_old); + private Optional selectActivity(String taskUuid) { + return dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), taskUuid); + } + + private List selectTaskCharecteristic(String taskUuid) { + return dbTester.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(dbTester.getSession(), Collections.singletonList(taskUuid)); + } + + private Optional selectTaskInput(String taskUuid) { + return dbTester.getDbClient().ceTaskInputDao().selectData(dbTester.getSession(), taskUuid); } private boolean scannerContextExists(String uuid) { @@ -99,7 +127,15 @@ public class PurgeCeActivitiesTest { CeActivityDto dto = new CeActivityDto(queueDto); dto.setStatus(CeActivityDto.Status.SUCCESS); when(system2.now()).thenReturn(date); + CeTaskCharacteristicDto ceTaskCharacteristicDto = new CeTaskCharacteristicDto() + .setUuid(UuidFactoryFast.getInstance().create()) + .setValue(randomAlphanumeric(10)) + .setKey(randomAlphanumeric(10)) + .setTaskUuid(dto.getUuid()); + + dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), dto.getUuid(), IOUtils.toInputStream(randomAlphanumeric(10), Charset.forName("UTF-8"))); dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto); + dbTester.getDbClient().ceTaskCharacteristicsDao().insert(dbTester.getSession(), Collections.singletonList(ceTaskCharacteristicDto)); dbTester.getSession().commit(); insertScannerContext(uuid, date); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java index 0eb7ca5583c..1a9f3547cd6 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java @@ -21,10 +21,12 @@ package org.sonar.db.ce; import java.util.Collection; import java.util.List; +import java.util.Set; import org.sonar.db.Dao; import org.sonar.db.DbSession; import static org.sonar.db.DatabaseUtils.executeLargeInputs; +import static org.sonar.db.DatabaseUtils.executeLargeUpdates; public class CeTaskCharacteristicDao implements Dao { @@ -38,6 +40,10 @@ public class CeTaskCharacteristicDao implements Dao { return executeLargeInputs(taskUuids, uuid -> mapper(dbSession).selectByTaskUuids(uuid)); } + public void deleteByTaskUuids(DbSession dbSession, Set taskUuids) { + executeLargeUpdates(taskUuids, mapper(dbSession)::deleteByTaskUuids); + } + private static CeTaskCharacteristicMapper mapper(DbSession session) { return session.getMapper(CeTaskCharacteristicMapper.class); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicMapper.java index 01fb720fd71..c4f0c6cabab 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicMapper.java @@ -28,4 +28,5 @@ public interface CeTaskCharacteristicMapper { void insert(CeTaskCharacteristicDto taskCharacteristic); + void deleteByTaskUuids(@Param("taskUuids") List taskUuids); } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeTaskCharacteristicMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeTaskCharacteristicMapper.xml index 942b031c50b..95fdfba037d 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeTaskCharacteristicMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeTaskCharacteristicMapper.xml @@ -34,4 +34,13 @@ ) + + DELETE + FROM ce_task_characteristics + WHERE + task_uuid IN + + #{taskUuid,jdbcType=VARCHAR} + + diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskCharacteristicDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskCharacteristicDaoTest.java index a49bdc84526..fb926328c7f 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskCharacteristicDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskCharacteristicDaoTest.java @@ -19,6 +19,7 @@ */ package org.sonar.db.ce; +import com.google.common.collect.ImmutableSet; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -26,6 +27,7 @@ import org.sonar.api.utils.System2; import org.sonar.db.DbTester; import static java.util.Arrays.asList; +import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; @@ -40,17 +42,9 @@ public class CeTaskCharacteristicDaoTest { @Test public void selectByTaskUuids() { - CeTaskCharacteristicDto dto1 = new CeTaskCharacteristicDto() - .setKey("key1") - .setValue("value1") - .setUuid("uuid1") - .setTaskUuid("task1"); - CeTaskCharacteristicDto dto2 = new CeTaskCharacteristicDto() - .setKey("key2") - .setValue("value2") - .setUuid("uuid2") - .setTaskUuid("task2"); - underTest.insert(dbTester.getSession(), asList(dto1, dto2)); + insert("key1", "value1", "uuid1", "task1"); + insert("key2", "value2", "uuid2", "task2"); + dbTester.getSession().commit(); assertThat(underTest.selectByTaskUuids(dbTester.getSession(), asList("task1", "task2"))) @@ -61,4 +55,35 @@ public class CeTaskCharacteristicDaoTest { assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("unknown"))).isEmpty(); } + + @Test + public void deleteByTaskUuids() { + insert("key1", "value1", "uuid1", "task1"); + insert("key2", "value2", "uuid2", "task2"); + insert("key3", "value3", "uuid3", "task3"); + + underTest.deleteByTaskUuids(dbTester.getSession(), ImmutableSet.of("task1", "task3")); + assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("task1"))).hasSize(0); + assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("task2"))).hasSize(1); + assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("task3"))).hasSize(0); + } + + @Test + public void deleteByTaskUuids_does_nothing_if_uuid_does_not_exist() { + insert("key1", "value1", "uuid1", "task1"); + + // must not fail + underTest.deleteByTaskUuids(dbTester.getSession(), singleton("task2")); + + assertThat(underTest.selectByTaskUuids(dbTester.getSession(), singletonList("task1"))).hasSize(1); + } + + private void insert(String key, String value, String uuid, String taskUuid) { + CeTaskCharacteristicDto dto1 = new CeTaskCharacteristicDto() + .setKey(key) + .setValue(value) + .setUuid(uuid) + .setTaskUuid(taskUuid); + underTest.insert(dbTester.getSession(), singleton(dto1)); + } } -- 2.39.5