From 3b5998a64f0c3eb8e42e48067b97c885739e874a Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Lesaint?= Date: Thu, 25 Aug 2016 15:45:56 +0200 Subject: [PATCH] SONAR-7911 restore purge of CE_ACTIVITIES + purge CE_SCANNER_CONTEXT --- .../container/ComputeEngineContainerImpl.java | 2 + .../computation/queue/PurgeCeActivities.java | 65 ++++++++++++++++++ .../computation/PurgeCeActivitiesTest.java | 67 +++++++++++++++++++ .../src/main/java/org/sonar/db/MyBatis.java | 4 +- .../java/org/sonar/db/ce/CeActivityDao.java | 7 +- .../org/sonar/db/ce/CeActivityMapper.java | 2 +- .../org/sonar/db/ce/CeScannerContextDao.java | 9 +++ .../sonar/db/ce/CeScannerContextMapper.java | 28 ++++++++ .../org/sonar/db/ce/CeActivityMapper.xml | 11 ++- .../sonar/db/ce/CeScannerContextMapper.xml | 11 +++ .../org/sonar/db/ce/CeActivityDaoTest.java | 12 ++-- .../sonar/db/ce/CeScannerContextDaoTest.java | 32 ++++++++- 12 files changed, 236 insertions(+), 14 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/queue/PurgeCeActivities.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/PurgeCeActivitiesTest.java create mode 100644 sonar-db/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java create mode 100644 sonar-db/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java index 7d6524dcf48..73e12ff3671 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java @@ -73,6 +73,7 @@ import org.sonar.server.activity.index.ActivityIndexer; import org.sonar.server.component.ComponentCleanerService; import org.sonar.server.component.ComponentFinder; import org.sonar.server.component.ComponentService; +import org.sonar.server.computation.queue.PurgeCeActivities; import org.sonar.server.computation.task.projectanalysis.ProjectAnalysisTaskModule; import org.sonar.server.computation.taskprocessor.CeTaskProcessorModule; import org.sonar.server.debt.DebtModelPluginRepository; @@ -660,6 +661,7 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer { // RegisterIssueFilters.class, DB maintenance, responsibility of Web Server // RenameIssueWidgets.class, UI related, anyway, DB maintenance, responsibility of Web Server ServerLifecycleNotifier.class, + PurgeCeActivities.class, // DisplayLogOnDeprecatedProjects.class, responsibility of Web Server // ClearRulesOverloadedDebt.class, DB maintenance, responsibility of Web Server }; diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/queue/PurgeCeActivities.java b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/PurgeCeActivities.java new file mode 100644 index 00000000000..84a68ff8bef --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/PurgeCeActivities.java @@ -0,0 +1,65 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.queue; + +import java.util.Calendar; +import java.util.Set; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.platform.Server; +import org.sonar.api.platform.ServerStartHandler; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.core.util.stream.Collectors; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.ce.CeActivityDto; + +@ComputeEngineSide +public class PurgeCeActivities implements ServerStartHandler { + + private static final Logger LOGGER = Loggers.get(PurgeCeActivities.class); + + private final DbClient dbClient; + private final System2 system2; + + public PurgeCeActivities(DbClient dbClient, System2 system2) { + this.dbClient = dbClient; + this.system2 = system2; + } + + @Override + public void onServerStart(Server server) { + try (DbSession dbSession = dbClient.openSession(false)) { + Calendar sixMonthsAgo = Calendar.getInstance(); + sixMonthsAgo.setTimeInMillis(system2.now()); + sixMonthsAgo.add(Calendar.DATE, -180); + + LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime()); + Set ceActivityUuids = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTimeInMillis()) + .stream() + .map(CeActivityDto::getUuid) + .collect(Collectors.toSet()); + dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids); + dbClient.ceScannerContextDao().deleteByUuids(dbSession, ceActivityUuids); + dbSession.commit(); + } + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/PurgeCeActivitiesTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/PurgeCeActivitiesTest.java new file mode 100644 index 00000000000..e6d9f24da32 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/PurgeCeActivitiesTest.java @@ -0,0 +1,67 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation; + +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.platform.Server; +import org.sonar.api.utils.internal.TestSystem2; +import org.sonar.db.DbTester; +import org.sonar.db.ce.CeActivityDto; +import org.sonar.db.ce.CeQueueDto; +import org.sonar.db.ce.CeTaskTypes; +import org.sonar.server.computation.queue.PurgeCeActivities; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +public class PurgeCeActivitiesTest { + + private TestSystem2 system2 = new TestSystem2(); + + @Rule + public DbTester dbTester = DbTester.create(system2); + + private PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2); + + @Test + public void delete_older_than_6_months() throws Exception { + insertWithDate("VERY_OLD", 1_000_000_000_000L); + insertWithDate("RECENT", 1_500_000_000_000L); + system2.setNow(1_500_000_000_100L); + + underTest.onServerStart(mock(Server.class)); + + assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "VERY_OLD").isPresent()).isFalse(); + assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "RECENT").isPresent()).isTrue(); + } + + private void insertWithDate(String uuid, long date) { + CeQueueDto queueDto = new CeQueueDto(); + queueDto.setUuid(uuid); + queueDto.setTaskType(CeTaskTypes.REPORT); + + CeActivityDto dto = new CeActivityDto(queueDto); + dto.setStatus(CeActivityDto.Status.SUCCESS); + system2.setNow(date); + dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto); + dbTester.getSession().commit(); + } +} diff --git a/sonar-db/src/main/java/org/sonar/db/MyBatis.java b/sonar-db/src/main/java/org/sonar/db/MyBatis.java index 9ea48e9e2cb..9bd59be0240 100644 --- a/sonar-db/src/main/java/org/sonar/db/MyBatis.java +++ b/sonar-db/src/main/java/org/sonar/db/MyBatis.java @@ -33,6 +33,7 @@ import org.sonar.db.activity.ActivityDto; import org.sonar.db.activity.ActivityMapper; import org.sonar.db.ce.CeActivityMapper; import org.sonar.db.ce.CeQueueMapper; +import org.sonar.db.ce.CeScannerContextMapper; import org.sonar.db.ce.CeTaskInputMapper; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentDtoWithSnapshotId; @@ -234,7 +235,8 @@ public class MyBatis { GroupMembershipMapper.class, QualityProfileMapper.class, ActiveRuleMapper.class, MeasureMapper.class, MetricMapper.class, CustomMeasureMapper.class, QualityGateMapper.class, QualityGateConditionMapper.class, ComponentMapper.class, SnapshotMapper.class, ProjectQgateAssociationMapper.class, EventMapper.class, - CeQueueMapper.class, CeActivityMapper.class, CeTaskInputMapper.class, ComponentLinkMapper.class, + CeQueueMapper.class, CeActivityMapper.class, CeTaskInputMapper.class, CeScannerContextMapper.class, + ComponentLinkMapper.class, Migration45Mapper.class, Migration50Mapper.class, Migration53Mapper.class }; confBuilder.loadMappers(mappers); diff --git a/sonar-db/src/main/java/org/sonar/db/ce/CeActivityDao.java b/sonar-db/src/main/java/org/sonar/db/ce/CeActivityDao.java index 91f155ee3ca..5a7cada22da 100644 --- a/sonar-db/src/main/java/org/sonar/db/ce/CeActivityDao.java +++ b/sonar-db/src/main/java/org/sonar/db/ce/CeActivityDao.java @@ -22,12 +22,15 @@ package org.sonar.db.ce; import com.google.common.base.Optional; import java.util.Collections; import java.util.List; +import java.util.Set; import javax.annotation.Nullable; import org.apache.ibatis.session.RowBounds; import org.sonar.api.utils.System2; import org.sonar.db.Dao; import org.sonar.db.DbSession; +import static org.sonar.db.DatabaseUtils.executeLargeUpdates; + public class CeActivityDao implements Dao { private final System2 system2; @@ -58,8 +61,8 @@ public class CeActivityDao implements Dao { return mapper(dbSession).selectOlderThan(beforeDate); } - public void deleteByUuid(DbSession dbSession, String uuid) { - mapper(dbSession).deleteByUuid(uuid); + public void deleteByUuids(DbSession dbSession, Set uuids) { + executeLargeUpdates(uuids, mapper(dbSession)::deleteByUuids); } /** diff --git a/sonar-db/src/main/java/org/sonar/db/ce/CeActivityMapper.java b/sonar-db/src/main/java/org/sonar/db/ce/CeActivityMapper.java index 8bdaa57b1af..ea5d9e9c885 100644 --- a/sonar-db/src/main/java/org/sonar/db/ce/CeActivityMapper.java +++ b/sonar-db/src/main/java/org/sonar/db/ce/CeActivityMapper.java @@ -46,5 +46,5 @@ public interface CeActivityMapper { void updateIsLastToTrueForUuid(@Param("uuid") String uuid, @Param("updatedAt") long updatedAt); - void deleteByUuid(@Param("uuid") String uuid); + void deleteByUuids(@Param("uuids") List uuids); } diff --git a/sonar-db/src/main/java/org/sonar/db/ce/CeScannerContextDao.java b/sonar-db/src/main/java/org/sonar/db/ce/CeScannerContextDao.java index 946eea53581..fd7dbdeec9f 100644 --- a/sonar-db/src/main/java/org/sonar/db/ce/CeScannerContextDao.java +++ b/sonar-db/src/main/java/org/sonar/db/ce/CeScannerContextDao.java @@ -26,11 +26,13 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collection; import java.util.Optional; import org.apache.commons.io.IOUtils; import org.sonar.api.utils.System2; import org.sonar.core.util.CloseableIterator; import org.sonar.db.Dao; +import org.sonar.db.DatabaseUtils; import org.sonar.db.DbSession; import static com.google.common.base.Preconditions.checkArgument; @@ -84,4 +86,11 @@ public class CeScannerContextDao implements Dao { } } + public void deleteByUuids(DbSession dbSession, Collection uuids) { + DatabaseUtils.executeLargeUpdates(uuids, mapper(dbSession)::deleteByUuids); + } + + private static CeScannerContextMapper mapper(DbSession dbSession) { + return dbSession.getMapper(CeScannerContextMapper.class); + } } diff --git a/sonar-db/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java b/sonar-db/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java new file mode 100644 index 00000000000..946246205aa --- /dev/null +++ b/sonar-db/src/main/java/org/sonar/db/ce/CeScannerContextMapper.java @@ -0,0 +1,28 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.ce; + +import java.util.List; +import org.apache.ibatis.annotations.Param; + +public interface CeScannerContextMapper { + + void deleteByUuids(@Param("uuids") List uuids); +} diff --git a/sonar-db/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml b/sonar-db/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml index 77d2ccd60f5..8ff5806d277 100644 --- a/sonar-db/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml +++ b/sonar-db/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml @@ -188,8 +188,13 @@ where uuid=#{uuid} - - delete from ce_activity - where uuid=#{uuid} + + delete + from ce_activity + where + uuid in + + #{uuid} + diff --git a/sonar-db/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml b/sonar-db/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml new file mode 100644 index 00000000000..f9ff2523926 --- /dev/null +++ b/sonar-db/src/main/resources/org/sonar/db/ce/CeScannerContextMapper.xml @@ -0,0 +1,11 @@ + + + + + + + delete from ce_scanner_context + where task_uuid in #{uuid} + + + diff --git a/sonar-db/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java b/sonar-db/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java index 45b8b41271a..51d31b7a4df 100644 --- a/sonar-db/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java +++ b/sonar-db/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java @@ -22,6 +22,7 @@ package org.sonar.db.ce; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; import java.util.Collections; import java.util.List; import javax.annotation.Nonnull; @@ -33,6 +34,7 @@ import org.sonar.core.util.stream.Collectors; import org.sonar.db.DbSession; import org.sonar.db.DbTester; +import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.sonar.db.ce.CeActivityDto.Status.FAILED; @@ -293,21 +295,23 @@ public class CeActivityDaoTest { } @Test - public void deleteByUuid() { + public void deleteByUuids() { insert("TASK_1", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS); insert("TASK_2", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS); + insert("TASK_3", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS); - underTest.deleteByUuid(db.getSession(), "TASK_1"); + underTest.deleteByUuids(db.getSession(), ImmutableSet.of("TASK_1", "TASK_3")); assertThat(underTest.selectByUuid(db.getSession(), "TASK_1").isPresent()).isFalse(); assertThat(underTest.selectByUuid(db.getSession(), "TASK_2").isPresent()).isTrue(); + assertThat(underTest.selectByUuid(db.getSession(), "TASK_3").isPresent()).isFalse(); } @Test - public void deleteByUuid_does_nothing_if_uuid_does_not_exist() { + public void deleteByUuids_does_nothing_if_uuid_does_not_exist() { insert("TASK_1", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS); // must not fail - underTest.deleteByUuid(db.getSession(), "TASK_2"); + underTest.deleteByUuids(db.getSession(), singleton("TASK_2")); assertThat(underTest.selectByUuid(db.getSession(), "TASK_1").isPresent()).isTrue(); } diff --git a/sonar-db/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java b/sonar-db/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java index e270a947532..b0c861c8181 100644 --- a/sonar-db/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java +++ b/sonar-db/src/test/java/org/sonar/db/ce/CeScannerContextDaoTest.java @@ -19,7 +19,7 @@ */ package org.sonar.db.ce; -import java.util.Collections; +import com.google.common.collect.ImmutableSet; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -29,6 +29,7 @@ import org.sonar.db.DbSession; import org.sonar.db.DbTester; import static java.lang.System.lineSeparator; +import static java.util.Collections.singleton; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -98,13 +99,38 @@ public class CeScannerContextDaoTest { public void insert_and_select_line_reader() { String scannerContext = "line 1" + lineSeparator() + "line 2" + lineSeparator() + "line 3"; underTest.insert(dbSession, SOME_UUID, scannerContextInputStreamOf(scannerContext)); - dbSession.commit(true); + dbSession.commit(); assertThat(underTest.selectScannerContext(dbSession, SOME_UUID)).contains(scannerContext); } + @Test + public void deleteByUuids_does_not_fail_on_empty_table() { + underTest.deleteByUuids(dbSession, singleton("some uuid")); + } + + @Test + public void deleteByUuids_deletes_specified_existing_uuids() { + insertScannerContext(SOME_UUID); + String data2 = insertScannerContext("UUID_2"); + insertScannerContext("UUID_3"); + + underTest.deleteByUuids(dbSession, ImmutableSet.of(SOME_UUID, "UUID_3", "UUID_4")); + + assertThat(underTest.selectScannerContext(dbSession, SOME_UUID)).isEmpty(); + assertThat(underTest.selectScannerContext(dbSession, "UUID_2")).contains(data2); + assertThat(underTest.selectScannerContext(dbSession, "UUID_3")).isEmpty(); + } + + private String insertScannerContext(String uuid) { + String data = "data of " + uuid; + underTest.insert(dbSession, uuid, scannerContextInputStreamOf(data)); + dbSession.commit(); + return data; + } + private static CloseableIterator scannerContextInputStreamOf(String data) { - return CloseableIterator.from(Collections.singleton(data).iterator()); + return CloseableIterator.from(singleton(data).iterator()); } } -- 2.39.5