From 96ab2f7679e5c904a942f5256969ddff07ca788d Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Mon, 28 Sep 2015 23:28:06 +0200 Subject: [PATCH] SONAR-6630 delete logs related to old reports --- .../server/computation/log/CeLogging.java | 5 +++ .../server/computation/log/LogFileRef.java | 2 +- .../computation/queue/PurgeCeActivities.java | 18 ++++++++--- .../computation/PurgeCeActivitiesTest.java | 12 +++++-- .../java/org/sonar/db/ce/CeActivityDao.java | 8 +++-- .../org/sonar/db/ce/CeActivityMapper.java | 4 ++- .../java/org/sonar/db/ce/CeTaskTypes.java | 2 +- .../org/sonar/db/ce/CeActivityMapper.xml | 11 +++++-- .../org/sonar/db/ce/CeActivityDaoTest.java | 32 ++++++++++++++----- 9 files changed, 72 insertions(+), 22 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogging.java b/server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogging.java index 55f76de99e8..c4f689bb557 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogging.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogging.java @@ -78,6 +78,11 @@ public class CeLogging { return Optional.absent(); } + public void deleteIfExists(LogFileRef ref) { + File logFile = new File(logsDir, ref.getRelativePath()); + logFile.delete(); + } + /** * Initialize logging of a Compute Engine task. Must be called * before first writing of log. diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/log/LogFileRef.java b/server/sonar-server/src/main/java/org/sonar/server/computation/log/LogFileRef.java index d530a51e89a..3378f4805e5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/log/LogFileRef.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/log/LogFileRef.java @@ -79,7 +79,7 @@ public class LogFileRef { if (!taskUuid.equals(that.taskUuid)) { return false; } - return !(componentUuid != null ? !componentUuid.equals(that.componentUuid) : that.componentUuid != null); + return componentUuid == null ? that.componentUuid == null : componentUuid.equals(that.componentUuid); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/queue/PurgeCeActivities.java b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/PurgeCeActivities.java index e92f7d15ef3..971e50ac3ea 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/queue/PurgeCeActivities.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/PurgeCeActivities.java @@ -20,6 +20,7 @@ package org.sonar.server.computation.queue; import java.util.Calendar; +import java.util.List; import org.sonar.api.platform.Server; import org.sonar.api.platform.ServerStartHandler; import org.sonar.api.server.ServerSide; @@ -28,6 +29,9 @@ import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.ce.CeActivityDto; +import org.sonar.server.computation.log.CeLogging; +import org.sonar.server.computation.log.LogFileRef; @ServerSide public class PurgeCeActivities implements ServerStartHandler { @@ -36,22 +40,28 @@ public class PurgeCeActivities implements ServerStartHandler { private final DbClient dbClient; private final System2 system2; + private final CeLogging ceLogging; - public PurgeCeActivities(DbClient dbClient, System2 system2) { + public PurgeCeActivities(DbClient dbClient, System2 system2, CeLogging ceLogging) { this.dbClient = dbClient; this.system2 = system2; + this.ceLogging = ceLogging; } @Override public void onServerStart(Server server) { - DbSession dbSession = dbClient.openSession(false); + DbSession dbSession = dbClient.openSession(true); try { Calendar sixMonthsAgo = Calendar.getInstance(); sixMonthsAgo.setTimeInMillis(system2.now()); sixMonthsAgo.add(Calendar.DATE, -180); - LOGGER.info("Delete the Compute Engine tasks created before " + sixMonthsAgo.getTime()); - dbClient.ceActivityDao().deleteOlderThan(dbSession, sixMonthsAgo.getTimeInMillis()); + LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime()); + List dtos = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTimeInMillis()); + for (CeActivityDto dto : dtos) { + dbClient.ceActivityDao().deleteByUuid(dbSession, dto.getUuid()); + ceLogging.deleteIfExists(LogFileRef.from(dto)); + } dbSession.commit(); } finally { diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/PurgeCeActivitiesTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/PurgeCeActivitiesTest.java index b31ae60d2a6..cc985811322 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/PurgeCeActivitiesTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/PurgeCeActivitiesTest.java @@ -26,10 +26,15 @@ import org.sonar.api.utils.internal.TestSystem2; import org.sonar.db.DbTester; import org.sonar.db.ce.CeActivityDto; import org.sonar.db.ce.CeQueueDto; +import org.sonar.db.ce.CeTaskTypes; +import org.sonar.server.computation.log.CeLogging; +import org.sonar.server.computation.log.LogFileRef; import org.sonar.server.computation.queue.PurgeCeActivities; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; public class PurgeCeActivitiesTest { @@ -37,7 +42,8 @@ public class PurgeCeActivitiesTest { @Rule public DbTester dbTester = DbTester.create(system2); - PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2); + CeLogging ceLogging = mock(CeLogging.class); + PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2, ceLogging); @Test public void delete_older_than_6_months() throws Exception { @@ -49,12 +55,14 @@ public class PurgeCeActivitiesTest { assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "VERY_OLD").isPresent()).isFalse(); assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "RECENT").isPresent()).isTrue(); + verify(ceLogging).deleteIfExists(new LogFileRef(CeTaskTypes.REPORT, "VERY_OLD", null)); + verify(ceLogging, never()).deleteIfExists(new LogFileRef(CeTaskTypes.REPORT, "RECENT", null)); } private void insertWithDate(String uuid, long date) { CeQueueDto queueDto = new CeQueueDto(); queueDto.setUuid(uuid); - queueDto.setTaskType("fake"); + queueDto.setTaskType(CeTaskTypes.REPORT); CeActivityDto dto = new CeActivityDto(queueDto); dto.setStatus(CeActivityDto.Status.SUCCESS); diff --git a/sonar-db/src/main/java/org/sonar/db/ce/CeActivityDao.java b/sonar-db/src/main/java/org/sonar/db/ce/CeActivityDao.java index 58214c0c0ad..f54d80f566d 100644 --- a/sonar-db/src/main/java/org/sonar/db/ce/CeActivityDao.java +++ b/sonar-db/src/main/java/org/sonar/db/ce/CeActivityDao.java @@ -52,8 +52,12 @@ public class CeActivityDao implements Dao { } } - public void deleteOlderThan(DbSession dbSession, long beforeDate) { - mapper(dbSession).deleteOlderThan(beforeDate); + public List selectOlderThan(DbSession dbSession, long beforeDate) { + return mapper(dbSession).selectOlderThan(beforeDate); + } + + public void deleteByUuid(DbSession dbSession, String uuid) { + mapper(dbSession).deleteByUuid(uuid); } /** diff --git a/sonar-db/src/main/java/org/sonar/db/ce/CeActivityMapper.java b/sonar-db/src/main/java/org/sonar/db/ce/CeActivityMapper.java index 7c019b25eb4..7d9ada5c11e 100644 --- a/sonar-db/src/main/java/org/sonar/db/ce/CeActivityMapper.java +++ b/sonar-db/src/main/java/org/sonar/db/ce/CeActivityMapper.java @@ -35,6 +35,8 @@ public interface CeActivityMapper { List selectByQuery(@Param("query") CeActivityQuery query, RowBounds rowBounds); + List selectOlderThan(@Param("beforeDate") long beforeDate); + int countByQuery(@Param("query") CeActivityQuery query); void insert(CeActivityDto dto); @@ -43,5 +45,5 @@ public interface CeActivityMapper { void updateIsLastToTrueForUuid(@Param("uuid") String uuid, @Param("updatedAt") long updatedAt); - void deleteOlderThan(@Param("beforeDate") long beforeDate); + void deleteByUuid(@Param("uuid") String uuid); } diff --git a/sonar-db/src/main/java/org/sonar/db/ce/CeTaskTypes.java b/sonar-db/src/main/java/org/sonar/db/ce/CeTaskTypes.java index 042ff5eec0e..3136e8295ca 100644 --- a/sonar-db/src/main/java/org/sonar/db/ce/CeTaskTypes.java +++ b/sonar-db/src/main/java/org/sonar/db/ce/CeTaskTypes.java @@ -19,7 +19,7 @@ */ package org.sonar.db.ce; -public class CeTaskTypes { +public final class CeTaskTypes { private CeTaskTypes() { // only statics diff --git a/sonar-db/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml b/sonar-db/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml index d6a13c87fac..c466d93cbcc 100644 --- a/sonar-db/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml +++ b/sonar-db/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml @@ -93,6 +93,12 @@ + + insert into ce_activity (uuid, component_uuid, status, task_type, is_last, is_last_key, submitter_login, submitted_at, started_at, @@ -128,9 +134,8 @@ where uuid=#{uuid} - + delete from ce_activity - where created_at < #{beforeDate,jdbcType=BIGINT} + where uuid=#{uuid} - diff --git a/sonar-db/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java b/sonar-db/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java index b52f6da2272..e714e688bf1 100644 --- a/sonar-db/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java +++ b/sonar-db/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java @@ -80,7 +80,7 @@ public class CeActivityDaoTest { } @Test - public void test_selectByQuery() throws Exception { + public void test_selectByQuery() { insert("TASK_1", REPORT, "PROJECT_1", CeActivityDto.Status.SUCCESS); insert("TASK_2", REPORT, "PROJECT_1", CeActivityDto.Status.FAILED); insert("TASK_3", REPORT, "PROJECT_2", CeActivityDto.Status.SUCCESS); @@ -116,7 +116,7 @@ public class CeActivityDaoTest { } @Test - public void test_countByQuery() throws Exception { + public void test_countByQuery() { insert("TASK_1", REPORT, "PROJECT_1", CeActivityDto.Status.SUCCESS); insert("TASK_2", REPORT, "PROJECT_1", CeActivityDto.Status.FAILED); insert("TASK_3", REPORT, "PROJECT_2", CeActivityDto.Status.SUCCESS); @@ -146,7 +146,7 @@ public class CeActivityDaoTest { } @Test - public void select_and_count_by_date() throws Exception { + public void select_and_count_by_date() { insertWithDates("UUID1", 1_450_000_000_000L, 1_470_000_000_000L); insertWithDates("UUID2", 1_460_000_000_000L, 1_480_000_000_000L); @@ -181,17 +181,33 @@ public class CeActivityDaoTest { } @Test - public void deleteOlderThan() throws Exception { + public void selectOlderThan() { insertWithCreationDate("TASK_1", 1_450_000_000_000L); insertWithCreationDate("TASK_2", 1_460_000_000_000L); insertWithCreationDate("TASK_3", 1_470_000_000_000L); - underTest.deleteOlderThan(db.getSession(), 1_465_000_000_000L); - db.getSession().commit(); + List dtos = underTest.selectOlderThan(db.getSession(), 1_465_000_000_000L); + assertThat(dtos).extracting("uuid").containsOnly("TASK_1", "TASK_2"); + } + + @Test + public void deleteByUuid() { + insert("TASK_1", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS); + insert("TASK_2", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS); + underTest.deleteByUuid(db.getSession(), "TASK_1"); assertThat(underTest.selectByUuid(db.getSession(), "TASK_1").isPresent()).isFalse(); - assertThat(underTest.selectByUuid(db.getSession(), "TASK_2").isPresent()).isFalse(); - assertThat(underTest.selectByUuid(db.getSession(), "TASK_3").isPresent()).isTrue(); + assertThat(underTest.selectByUuid(db.getSession(), "TASK_2").isPresent()).isTrue(); + } + + @Test + public void deleteByUuid_does_nothing_if_uuid_does_not_exist() { + insert("TASK_1", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS); + + // must not fail + underTest.deleteByUuid(db.getSession(), "TASK_2"); + + assertThat(underTest.selectByUuid(db.getSession(), "TASK_1").isPresent()).isTrue(); } private void insert(String uuid, String type, String componentUuid, CeActivityDto.Status status) { -- 2.39.5