import org.sonar.server.component.ComponentCleanerService;
import org.sonar.server.component.ComponentFinder;
import org.sonar.server.component.ComponentService;
+import org.sonar.server.computation.queue.PurgeCeActivities;
import org.sonar.server.computation.task.projectanalysis.ProjectAnalysisTaskModule;
import org.sonar.server.computation.taskprocessor.CeTaskProcessorModule;
import org.sonar.server.debt.DebtModelPluginRepository;
// RegisterIssueFilters.class, DB maintenance, responsibility of Web Server
// RenameIssueWidgets.class, UI related, anyway, DB maintenance, responsibility of Web Server
ServerLifecycleNotifier.class,
+ PurgeCeActivities.class,
// DisplayLogOnDeprecatedProjects.class, responsibility of Web Server
// ClearRulesOverloadedDebt.class, DB maintenance, responsibility of Web Server
};
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.computation.queue;
+
+import java.util.Calendar;
+import java.util.Set;
+import org.sonar.api.ce.ComputeEngineSide;
+import org.sonar.api.platform.Server;
+import org.sonar.api.platform.ServerStartHandler;
+import org.sonar.api.utils.System2;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.core.util.stream.Collectors;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.ce.CeActivityDto;
+
+@ComputeEngineSide
+public class PurgeCeActivities implements ServerStartHandler {
+
+ private static final Logger LOGGER = Loggers.get(PurgeCeActivities.class);
+
+ private final DbClient dbClient;
+ private final System2 system2;
+
+ public PurgeCeActivities(DbClient dbClient, System2 system2) {
+ this.dbClient = dbClient;
+ this.system2 = system2;
+ }
+
+ @Override
+ public void onServerStart(Server server) {
+ try (DbSession dbSession = dbClient.openSession(false)) {
+ Calendar sixMonthsAgo = Calendar.getInstance();
+ sixMonthsAgo.setTimeInMillis(system2.now());
+ sixMonthsAgo.add(Calendar.DATE, -180);
+
+ LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime());
+ Set<String> ceActivityUuids = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTimeInMillis())
+ .stream()
+ .map(CeActivityDto::getUuid)
+ .collect(Collectors.toSet());
+ dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids);
+ dbClient.ceScannerContextDao().deleteByUuids(dbSession, ceActivityUuids);
+ dbSession.commit();
+ }
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.computation;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.platform.Server;
+import org.sonar.api.utils.internal.TestSystem2;
+import org.sonar.db.DbTester;
+import org.sonar.db.ce.CeActivityDto;
+import org.sonar.db.ce.CeQueueDto;
+import org.sonar.db.ce.CeTaskTypes;
+import org.sonar.server.computation.queue.PurgeCeActivities;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+
+public class PurgeCeActivitiesTest {
+
+ private TestSystem2 system2 = new TestSystem2();
+
+ @Rule
+ public DbTester dbTester = DbTester.create(system2);
+
+ private PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2);
+
+ @Test
+ public void delete_older_than_6_months() throws Exception {
+ insertWithDate("VERY_OLD", 1_000_000_000_000L);
+ insertWithDate("RECENT", 1_500_000_000_000L);
+ system2.setNow(1_500_000_000_100L);
+
+ underTest.onServerStart(mock(Server.class));
+
+ assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "VERY_OLD").isPresent()).isFalse();
+ assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "RECENT").isPresent()).isTrue();
+ }
+
+ private void insertWithDate(String uuid, long date) {
+ CeQueueDto queueDto = new CeQueueDto();
+ queueDto.setUuid(uuid);
+ queueDto.setTaskType(CeTaskTypes.REPORT);
+
+ CeActivityDto dto = new CeActivityDto(queueDto);
+ dto.setStatus(CeActivityDto.Status.SUCCESS);
+ system2.setNow(date);
+ dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto);
+ dbTester.getSession().commit();
+ }
+}
import org.sonar.db.activity.ActivityMapper;
import org.sonar.db.ce.CeActivityMapper;
import org.sonar.db.ce.CeQueueMapper;
+import org.sonar.db.ce.CeScannerContextMapper;
import org.sonar.db.ce.CeTaskInputMapper;
import org.sonar.db.component.ComponentDto;
import org.sonar.db.component.ComponentDtoWithSnapshotId;
GroupMembershipMapper.class, QualityProfileMapper.class, ActiveRuleMapper.class,
MeasureMapper.class, MetricMapper.class, CustomMeasureMapper.class, QualityGateMapper.class, QualityGateConditionMapper.class, ComponentMapper.class, SnapshotMapper.class,
ProjectQgateAssociationMapper.class, EventMapper.class,
- CeQueueMapper.class, CeActivityMapper.class, CeTaskInputMapper.class, ComponentLinkMapper.class,
+ CeQueueMapper.class, CeActivityMapper.class, CeTaskInputMapper.class, CeScannerContextMapper.class,
+ ComponentLinkMapper.class,
Migration45Mapper.class, Migration50Mapper.class, Migration53Mapper.class
};
confBuilder.loadMappers(mappers);
import com.google.common.base.Optional;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import javax.annotation.Nullable;
import org.apache.ibatis.session.RowBounds;
import org.sonar.api.utils.System2;
import org.sonar.db.Dao;
import org.sonar.db.DbSession;
+import static org.sonar.db.DatabaseUtils.executeLargeUpdates;
+
public class CeActivityDao implements Dao {
private final System2 system2;
return mapper(dbSession).selectOlderThan(beforeDate);
}
- public void deleteByUuid(DbSession dbSession, String uuid) {
- mapper(dbSession).deleteByUuid(uuid);
+ public void deleteByUuids(DbSession dbSession, Set<String> uuids) {
+ executeLargeUpdates(uuids, mapper(dbSession)::deleteByUuids);
}
/**
void updateIsLastToTrueForUuid(@Param("uuid") String uuid, @Param("updatedAt") long updatedAt);
- void deleteByUuid(@Param("uuid") String uuid);
+ void deleteByUuids(@Param("uuids") List<String> uuids);
}
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.Collection;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.sonar.api.utils.System2;
import org.sonar.core.util.CloseableIterator;
import org.sonar.db.Dao;
+import org.sonar.db.DatabaseUtils;
import org.sonar.db.DbSession;
import static com.google.common.base.Preconditions.checkArgument;
}
}
+ public void deleteByUuids(DbSession dbSession, Collection<String> uuids) {
+ DatabaseUtils.executeLargeUpdates(uuids, mapper(dbSession)::deleteByUuids);
+ }
+
+ private static CeScannerContextMapper mapper(DbSession dbSession) {
+ return dbSession.getMapper(CeScannerContextMapper.class);
+ }
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.db.ce;
+
+import java.util.List;
+import org.apache.ibatis.annotations.Param;
+
+public interface CeScannerContextMapper {
+
+ void deleteByUuids(@Param("uuids") List<String> uuids);
+}
where uuid=#{uuid}
</update>
- <delete id="deleteByUuid" parameterType="string">
- delete from ce_activity
- where uuid=#{uuid}
+ <delete id="deleteByUuids" parameterType="string">
+ delete
+ from ce_activity
+ where
+ uuid in
+ <foreach collection="uuids" open="(" close=")" item="uuid" separator=",">
+ #{uuid}
+ </foreach>
</delete>
</mapper>
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="org.sonar.db.ce.CeScannerContextMapper">
+
+ <delete id="deleteByUuids" parameterType="String">
+ delete from ce_scanner_context
+ where task_uuid in <foreach collection="uuids" open="(" close=")" item="uuid" separator=",">#{uuid}</foreach>
+ </delete>
+
+</mapper>
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;
+import static java.util.Collections.singleton;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.sonar.db.ce.CeActivityDto.Status.FAILED;
}
@Test
- public void deleteByUuid() {
+ public void deleteByUuids() {
insert("TASK_1", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS);
insert("TASK_2", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS);
+ insert("TASK_3", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS);
- underTest.deleteByUuid(db.getSession(), "TASK_1");
+ underTest.deleteByUuids(db.getSession(), ImmutableSet.of("TASK_1", "TASK_3"));
assertThat(underTest.selectByUuid(db.getSession(), "TASK_1").isPresent()).isFalse();
assertThat(underTest.selectByUuid(db.getSession(), "TASK_2").isPresent()).isTrue();
+ assertThat(underTest.selectByUuid(db.getSession(), "TASK_3").isPresent()).isFalse();
}
@Test
- public void deleteByUuid_does_nothing_if_uuid_does_not_exist() {
+ public void deleteByUuids_does_nothing_if_uuid_does_not_exist() {
insert("TASK_1", "REPORT", "COMPONENT1", CeActivityDto.Status.SUCCESS);
// must not fail
- underTest.deleteByUuid(db.getSession(), "TASK_2");
+ underTest.deleteByUuids(db.getSession(), singleton("TASK_2"));
assertThat(underTest.selectByUuid(db.getSession(), "TASK_1").isPresent()).isTrue();
}
*/
package org.sonar.db.ce;
-import java.util.Collections;
+import com.google.common.collect.ImmutableSet;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.sonar.db.DbTester;
import static java.lang.System.lineSeparator;
+import static java.util.Collections.singleton;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
public void insert_and_select_line_reader() {
String scannerContext = "line 1" + lineSeparator() + "line 2" + lineSeparator() + "line 3";
underTest.insert(dbSession, SOME_UUID, scannerContextInputStreamOf(scannerContext));
- dbSession.commit(true);
+ dbSession.commit();
assertThat(underTest.selectScannerContext(dbSession, SOME_UUID)).contains(scannerContext);
}
+ @Test
+ public void deleteByUuids_does_not_fail_on_empty_table() {
+ underTest.deleteByUuids(dbSession, singleton("some uuid"));
+ }
+
+ @Test
+ public void deleteByUuids_deletes_specified_existing_uuids() {
+ insertScannerContext(SOME_UUID);
+ String data2 = insertScannerContext("UUID_2");
+ insertScannerContext("UUID_3");
+
+ underTest.deleteByUuids(dbSession, ImmutableSet.of(SOME_UUID, "UUID_3", "UUID_4"));
+
+ assertThat(underTest.selectScannerContext(dbSession, SOME_UUID)).isEmpty();
+ assertThat(underTest.selectScannerContext(dbSession, "UUID_2")).contains(data2);
+ assertThat(underTest.selectScannerContext(dbSession, "UUID_3")).isEmpty();
+ }
+
+ private String insertScannerContext(String uuid) {
+ String data = "data of " + uuid;
+ underTest.insert(dbSession, uuid, scannerContextInputStreamOf(data));
+ dbSession.commit();
+ return data;
+ }
+
private static CloseableIterator<String> scannerContextInputStreamOf(String data) {
- return CloseableIterator.from(Collections.singleton(data).iterator());
+ return CloseableIterator.from(singleton(data).iterator());
}
}
projects / sonarqube.git / commitdiff