From 565791c16e84b8774bbf72104b7e66e0d372c5b1 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Fri, 24 May 2013 16:21:14 +0200 Subject: SONAR-4308 Update the DBCleaner mechanism to purge closed issues after X weeks --- .../org/sonar/core/purge/PurgeConfiguration.java | 64 ++++++++++++++++++++ .../main/java/org/sonar/core/purge/PurgeDao.java | 45 ++++++++------ .../java/org/sonar/core/purge/PurgeMapper.java | 5 ++ .../resources/org/sonar/core/purge/PurgeMapper.xml | 61 +++++++++++++++++++ .../sonar/core/purge/PurgeConfigurationTest.java | 50 ++++++++++++++++ .../java/org/sonar/core/purge/PurgeDaoTest.java | 31 +++++++--- .../should_delete_all_closed_issues-result.xml | 69 ++++++++++++++++++++++ .../should_delete_all_closed_issues.xml | 62 +++++++++++++++++++ .../should_delete_old_closed_issues-result.xml | 63 ++++++++++++++++++++ .../should_delete_old_closed_issues.xml | 62 +++++++++++++++++++ 10 files changed, 485 insertions(+), 27 deletions(-) create mode 100644 sonar-core/src/main/java/org/sonar/core/purge/PurgeConfiguration.java create mode 100644 sonar-core/src/test/java/org/sonar/core/purge/PurgeConfigurationTest.java create mode 100644 sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_all_closed_issues-result.xml create mode 100644 sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_all_closed_issues.xml create mode 100644 sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_old_closed_issues-result.xml create mode 100644 sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_old_closed_issues.xml (limited to 'sonar-core/src') diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeConfiguration.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeConfiguration.java new file mode 100644 index 00000000000..6f00448d4d9 --- /dev/null +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeConfiguration.java @@ -0,0 +1,64 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.core.purge; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.time.DateUtils; + +import javax.annotation.CheckForNull; +import java.util.Date; + +public class PurgeConfiguration { + + private static final int ONE_DAY_IN_MS = 24 * 60 * 60 * 1000; + + private final long rootProjectId; + private final String[] scopesWithoutHistoricalData; + private final int maxAgeInDaysOfClosedIssues; + + public PurgeConfiguration(long rootProjectId, String[] scopesWithoutHistoricalData, int maxAgeInDaysOfClosedIssues) { + this.rootProjectId = rootProjectId; + this.scopesWithoutHistoricalData = scopesWithoutHistoricalData; + this.maxAgeInDaysOfClosedIssues = maxAgeInDaysOfClosedIssues; + } + + public long rootProjectId() { + return rootProjectId; + } + + public String[] scopesWithoutHistoricalData() { + return scopesWithoutHistoricalData; + } + + @CheckForNull + public Date maxLiveDateOfClosedIssues() { + return maxLiveDateOfClosedIssues(new Date()); + } + + @VisibleForTesting + Date maxLiveDateOfClosedIssues(Date now) { + if (maxAgeInDaysOfClosedIssues > 0) { + return DateUtils.addDays(now, -maxAgeInDaysOfClosedIssues); + } + + // delete all closed issues + return null; + } +} diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java index 6f25476627a..9c848aabf23 100644 --- a/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java @@ -32,6 +32,7 @@ import org.sonar.core.resource.ResourceDao; import org.sonar.core.resource.ResourceDto; import java.util.Collections; +import java.util.Date; import java.util.List; /** @@ -49,56 +50,65 @@ public class PurgeDao { this.profiler = profiler; } - public PurgeDao purge(long rootResourceId, String[] scopesWithoutHistoricalData) { + + public PurgeDao purge(PurgeConfiguration conf) { SqlSession session = mybatis.openBatchSession(); - PurgeMapper purgeMapper = session.getMapper(PurgeMapper.class); - PurgeCommands commands = new PurgeCommands(session, purgeMapper, profiler); + PurgeMapper mapper = session.getMapper(PurgeMapper.class); + PurgeCommands commands = new PurgeCommands(session, mapper, profiler); try { - List projects = getProjects(rootResourceId, session); + List projects = getProjects(conf.rootProjectId(), session); for (ResourceDto project : projects) { LOG.info("-> Clean " + project.getLongName() + " [id=" + project.getId() + "]"); deleteAbortedBuilds(project, commands); - purge(project, scopesWithoutHistoricalData, commands); + purge(project, conf.scopesWithoutHistoricalData(), commands); } for (ResourceDto project : projects) { - disableOrphanResources(project, session, purgeMapper); + disableOrphanResources(project, session, mapper); } + deleteOldClosedIssues(conf, mapper); + session.commit(); } finally { MyBatis.closeQuietly(session); } return this; } + private void deleteOldClosedIssues(PurgeConfiguration conf, PurgeMapper mapper) { + Date toDate = conf.maxLiveDateOfClosedIssues(); + mapper.deleteOldClosedIssueChanges(conf.rootProjectId(), toDate); + mapper.deleteOldClosedIssues(conf.rootProjectId(), toDate); + } + private void deleteAbortedBuilds(ResourceDto project, PurgeCommands commands) { if (hasAbortedBuilds(project.getId(), commands)) { LOG.info("<- Delete aborted builds"); PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setStatus(new String[] {"U"}) - .setRootProjectId(project.getId()); + .setIslast(false) + .setStatus(new String[]{"U"}) + .setRootProjectId(project.getId()); commands.deleteSnapshots(query); } } private boolean hasAbortedBuilds(Long projectId, PurgeCommands commands) { PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setStatus(new String[] {"U"}) - .setResourceId(projectId); + .setIslast(false) + .setStatus(new String[]{"U"}) + .setResourceId(projectId); return !commands.selectSnapshotIds(query).isEmpty(); } private void purge(ResourceDto project, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) { List projectSnapshotIds = purgeCommands.selectSnapshotIds( - PurgeSnapshotQuery.create().setResourceId(project.getId()).setIslast(false).setNotPurged(true) - ); + PurgeSnapshotQuery.create().setResourceId(project.getId()).setIslast(false).setNotPurged(true) + ); for (final Long projectSnapshotId : projectSnapshotIds) { LOG.info("<- Clean snapshot " + projectSnapshotId); if (!ArrayUtils.isEmpty(scopesWithoutHistoricalData)) { PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setScopes(scopesWithoutHistoricalData) - .setRootSnapshotId(projectSnapshotId); + .setIslast(false) + .setScopes(scopesWithoutHistoricalData) + .setRootSnapshotId(projectSnapshotId); purgeCommands.deleteSnapshots(query); } @@ -163,7 +173,6 @@ public class PurgeDao { mapper.deleteResourceIndex(resourceId); mapper.setSnapshotIsLastToFalse(resourceId); mapper.disableResource(resourceId); - //TODO mapper.closeResourceReviews(resourceId); } public PurgeDao deleteSnapshots(PurgeSnapshotQuery query) { diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java index 1c3c282fd1c..66b1550780e 100644 --- a/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java @@ -21,6 +21,8 @@ package org.sonar.core.purge; import org.apache.ibatis.annotations.Param; +import javax.annotation.Nullable; +import java.util.Date; import java.util.List; public interface PurgeMapper { @@ -95,4 +97,7 @@ public interface PurgeMapper { void deleteResourceIssues(long resourceId); + void deleteOldClosedIssueChanges(@Param("rootProjectId") long rootProjectId, @Nullable @Param("toDate") Date toDate); + + void deleteOldClosedIssues(@Param("rootProjectId") long rootProjectId, @Nullable @Param("toDate") Date toDate); } diff --git a/sonar-core/src/main/resources/org/sonar/core/purge/PurgeMapper.xml b/sonar-core/src/main/resources/org/sonar/core/purge/PurgeMapper.xml index ab1e532a38f..1a38dae3827 100644 --- a/sonar-core/src/main/resources/org/sonar/core/purge/PurgeMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/purge/PurgeMapper.xml @@ -215,5 +215,66 @@ delete from issues where resource_id=#{id} + + + delete from issue_changes ic + where exists ( + select * from issues i + where i.project_id=#{rootProjectId} and i.kee=ic.issue_key + + + and i.issue_close_date is not null + + + and i.issue_close_date < #{toDate} + + + ) + + + + + delete issue_changes from issue_changes + inner join issues on issue_changes.issue_key=issues.kee + where issues.project_id=#{rootProjectId} + + + and issues.issue_close_date is not null + + + and issues.issue_close_date < #{toDate} + + + + + + + delete ic + from issue_changes as ic, issues as i + where i.project_id=#{rootProjectId} + and ic.issue_key=i.kee + + + and i.issue_close_date is not null + + + and i.issue_close_date < #{toDate} + + + + + + delete from issues + where project_id=#{rootProjectId} + + + and issue_close_date is not null + + + and issue_close_date < #{toDate} + + + + diff --git a/sonar-core/src/test/java/org/sonar/core/purge/PurgeConfigurationTest.java b/sonar-core/src/test/java/org/sonar/core/purge/PurgeConfigurationTest.java new file mode 100644 index 00000000000..d1647a84341 --- /dev/null +++ b/sonar-core/src/test/java/org/sonar/core/purge/PurgeConfigurationTest.java @@ -0,0 +1,50 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.core.purge; + +import org.junit.Test; +import org.sonar.api.utils.DateUtils; + +import java.util.Date; + +import static org.fest.assertions.Assertions.assertThat; + +public class PurgeConfigurationTest { + @Test + public void should_delete_all_closed_issues() throws Exception { + PurgeConfiguration conf = new PurgeConfiguration(1L, new String[0], 0); + assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); + + conf = new PurgeConfiguration(1L, new String[0], -1); + assertThat(conf.maxLiveDateOfClosedIssues()).isNull(); + } + + @Test + public void should_delete_only_old_closed_issues() throws Exception { + Date now = DateUtils.parseDate("2013-05-18"); + + PurgeConfiguration conf = new PurgeConfiguration(1L, new String[0], 30); + Date toDate = conf.maxLiveDateOfClosedIssues(now); + + assertThat(toDate.getYear()).isEqualTo(113);//=2013 + assertThat(toDate.getMonth()).isEqualTo(3); // means April + assertThat(toDate.getDate()).isEqualTo(18); + } +} diff --git a/sonar-core/src/test/java/org/sonar/core/purge/PurgeDaoTest.java b/sonar-core/src/test/java/org/sonar/core/purge/PurgeDaoTest.java index f629da7b04c..5f09193bf75 100644 --- a/sonar-core/src/test/java/org/sonar/core/purge/PurgeDaoTest.java +++ b/sonar-core/src/test/java/org/sonar/core/purge/PurgeDaoTest.java @@ -19,14 +19,12 @@ */ package org.sonar.core.purge; -import org.apache.ibatis.session.SqlSession; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.junit.Before; import org.junit.Test; import org.sonar.api.resources.Scopes; import org.sonar.core.persistence.AbstractDaoTestCase; -import org.sonar.core.persistence.MyBatis; import org.sonar.core.resource.ResourceDao; import java.util.List; @@ -47,28 +45,28 @@ public class PurgeDaoTest extends AbstractDaoTestCase { @Test public void shouldDeleteAbortedBuilds() { setupData("shouldDeleteAbortedBuilds"); - dao.purge(1L, new String[0]); + dao.purge(new PurgeConfiguration(1L, new String[0], 30)); checkTables("shouldDeleteAbortedBuilds", "snapshots"); } @Test public void shouldPurgeProject() { setupData("shouldPurgeProject"); - dao.purge(1, new String[0]); + dao.purge(new PurgeConfiguration(1L, new String[0], 30)); checkTables("shouldPurgeProject", "projects", "snapshots"); } @Test public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() { setupData("shouldDeleteHistoricalDataOfDirectoriesAndFiles"); - dao.purge(1, new String[] {Scopes.DIRECTORY, Scopes.FILE}); + dao.purge(new PurgeConfiguration(1L, new String[]{Scopes.DIRECTORY, Scopes.FILE}, 30)); checkTables("shouldDeleteHistoricalDataOfDirectoriesAndFiles", "projects", "snapshots"); } @Test public void shouldDisableResourcesWithoutLastSnapshot() { setupData("shouldDisableResourcesWithoutLastSnapshot"); - dao.purge(1, new String[0]); + dao.purge(new PurgeConfiguration(1L, new String[0], 30)); checkTables("shouldDisableResourcesWithoutLastSnapshot", "projects", "snapshots"); } @@ -97,6 +95,21 @@ public class PurgeDaoTest extends AbstractDaoTestCase { assertEmptyTables("projects", "snapshots", "action_plans", "issues", "issue_changes"); } + @Test + public void should_delete_old_closed_issues() { + setupData("should_delete_old_closed_issues"); + dao.purge(new PurgeConfiguration(1L, new String[0], 30)); + checkTables("should_delete_old_closed_issues", "issues", "issue_changes"); + } + + @Test + public void should_delete_all_closed_issues() { + setupData("should_delete_all_closed_issues"); + dao.purge(new PurgeConfiguration(1L, new String[0], 0)); + checkTables("should_delete_all_closed_issues", "issues", "issue_changes"); + } + + static final class SnapshotMatcher extends BaseMatcher { long snapshotId; boolean isLast; @@ -115,9 +128,9 @@ public class PurgeDaoTest extends AbstractDaoTestCase { public void describeTo(Description description) { description - .appendText("snapshotId").appendValue(snapshotId) - .appendText("isLast").appendValue(isLast) - .appendText("hasEvents").appendValue(hasEvents); + .appendText("snapshotId").appendValue(snapshotId) + .appendText("isLast").appendValue(isLast) + .appendText("hasEvents").appendValue(hasEvents); } } } diff --git a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_all_closed_issues-result.xml b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_all_closed_issues-result.xml new file mode 100644 index 00000000000..4fac1341e9b --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_all_closed_issues-result.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_all_closed_issues.xml b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_all_closed_issues.xml new file mode 100644 index 00000000000..d73273345a4 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_all_closed_issues.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_old_closed_issues-result.xml b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_old_closed_issues-result.xml new file mode 100644 index 00000000000..c68ff575f93 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_old_closed_issues-result.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_old_closed_issues.xml b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_old_closed_issues.xml new file mode 100644 index 00000000000..3fc561c92d4 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/should_delete_old_closed_issues.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file -- cgit v1.2.3