From 0b81985900032b040683832a2ab3176b91b0c6f1 Mon Sep 17 00:00:00 2001 From: Teryk Bellahsene Date: Thu, 30 Oct 2014 10:43:20 +0100 Subject: [PATCH] SONAR-5657 - Compute Engine - Delete old closed issues from index --- .../computation/ComputationStepRegistry.java | 4 +- ...bCleanerStep.java => DataCleanerStep.java} | 21 ++++++++-- .../sonar/server/issue/index/IssueIndex.java | 11 ++++- .../server/platform/ServerComponents.java | 2 +- .../ComputationStepRegistryTest.java | 8 ++-- ...StepTest.java => DataCleanerStepTest.java} | 17 ++++++-- .../server/computation/DbCleanStepTest.java | 41 ------------------- .../issue/index/IssueIndexMediumTest.java | 27 ++++++++++++ .../dbcleaner/DefaultPurgeTask.java | 19 ++++----- .../sonar/core/purge/PurgeConfiguration.java | 11 +++++ 10 files changed, 93 insertions(+), 68 deletions(-) rename server/sonar-server/src/main/java/org/sonar/server/computation/{DbCleanerStep.java => DataCleanerStep.java} (61%) rename server/sonar-server/src/test/java/org/sonar/server/computation/{DbCleanerStepTest.java => DataCleanerStepTest.java} (77%) delete mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/DbCleanStepTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/ComputationStepRegistry.java b/server/sonar-server/src/main/java/org/sonar/server/computation/ComputationStepRegistry.java index 08fc8e2076b..00548e7eb42 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/ComputationStepRegistry.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/ComputationStepRegistry.java @@ -34,8 +34,8 @@ public class ComputationStepRegistry implements ServerComponent { SynchronizeProjectPermissionsStep synchronizeProjectPermissionsStep, IndexProjectIssuesStep indexProjectIssuesStep, SwitchSnapshotStep switchSnapshotStep, - DbCleanerStep dbCleanerStep) { - steps = ImmutableList.of(getAndSetProjectStep, synchronizeProjectPermissionsStep, switchSnapshotStep, dbCleanerStep, indexProjectIssuesStep); + DataCleanerStep dataCleanerStep) { + steps = ImmutableList.of(getAndSetProjectStep, synchronizeProjectPermissionsStep, switchSnapshotStep, dataCleanerStep, indexProjectIssuesStep); } public List steps() { diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/DbCleanerStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/DataCleanerStep.java similarity index 61% rename from server/sonar-server/src/main/java/org/sonar/server/computation/DbCleanerStep.java rename to server/sonar-server/src/main/java/org/sonar/server/computation/DataCleanerStep.java index 7b9be40d236..7d87bbef0ac 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/DbCleanerStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/DataCleanerStep.java @@ -20,20 +20,35 @@ package org.sonar.server.computation; +import org.sonar.api.config.Settings; import org.sonar.core.computation.db.AnalysisReportDto; import org.sonar.core.computation.dbcleaner.DefaultPurgeTask; import org.sonar.core.persistence.DbSession; +import org.sonar.core.purge.PurgeConfiguration; +import org.sonar.server.issue.index.IssueIndex; -public class DbCleanerStep implements ComputationStep { +import java.util.Date; + +public class DataCleanerStep implements ComputationStep { private final DefaultPurgeTask purgeTask; + private final IssueIndex issueIndex; + private final Settings settings; - public DbCleanerStep(DefaultPurgeTask purgeTask) { + public DataCleanerStep(DefaultPurgeTask purgeTask, IssueIndex issueIndex, Settings settings) { this.purgeTask = purgeTask; + this.issueIndex = issueIndex; + this.settings = settings; } @Override public void execute(DbSession session, AnalysisReportDto report) { - purgeTask.purge(report.getProject().getId()); + Long projectId = report.getProject().getId(); + purgeTask.purge(projectId); + issueIndex.deleteClosedIssuesOfProjectBefore(report.getProjectUuid(), deleteIssuesBeforeThisDate(projectId)); + } + + private Date deleteIssuesBeforeThisDate(Long resourceId) { + return PurgeConfiguration.newDefaultPurgeConfiguration(resourceId, settings).maxLiveDateOfClosedIssues(); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 56b0c503390..424b7616e00 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -226,8 +226,15 @@ public class IssueIndex extends BaseIndex { getClient().prepareDeleteByQuery(getIndexName()).setQuery(queryBuilder).get(); } - public void deleteByProjectUuidBefore(String uuid, Date beforeDate) { - // TODO to implement + public void deleteClosedIssuesOfProjectBefore(String uuid, Date beforeDate) { + FilterBuilder projectFilter = FilterBuilders.boolFilter().must(FilterBuilders.termsFilter(IssueNormalizer.IssueField.PROJECT.field(), uuid)); + FilterBuilder dateFilter = FilterBuilders.rangeFilter(IssueNormalizer.IssueField.ISSUE_CLOSE_DATE.field()).lt(beforeDate.getTime()); + QueryBuilder queryBuilder = QueryBuilders.filteredQuery( + QueryBuilders.matchAllQuery(), + FilterBuilders.andFilter(projectFilter, dateFilter) + ); + + getClient().prepareDeleteByQuery(getIndexName()).setQuery(queryBuilder).get(); } /* Build main filter (match based) */ diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java index fef53361a13..0fec3f77e6f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java @@ -594,7 +594,7 @@ class ServerComponents { pico.addSingleton(SynchronizeProjectPermissionsStep.class); pico.addSingleton(IndexProjectIssuesStep.class); pico.addSingleton(SwitchSnapshotStep.class); - pico.addSingleton(DbCleanerStep.class); + pico.addSingleton(DataCleanerStep.class); pico.add(AnalysisReportQueue.class); pico.addSingleton(AnalysisReportTaskLauncher.class); pico.addSingleton(AnalysisReportWebService.class); diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/ComputationStepRegistryTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/ComputationStepRegistryTest.java index c2f56da319a..f2cdc8ce10d 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/ComputationStepRegistryTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/ComputationStepRegistryTest.java @@ -33,7 +33,7 @@ public class ComputationStepRegistryTest { private SynchronizeProjectPermissionsStep synchronizeProjectPermissionsStep; private IndexProjectIssuesStep indexProjectIssuesStep; private SwitchSnapshotStep switchSnapshotStep; - private DbCleanerStep dbCleanerStep; + private DataCleanerStep dataCleanerStep; @Before public void before() { @@ -41,13 +41,13 @@ public class ComputationStepRegistryTest { synchronizeProjectPermissionsStep = mock(SynchronizeProjectPermissionsStep.class); indexProjectIssuesStep = mock(IndexProjectIssuesStep.class); switchSnapshotStep = mock(SwitchSnapshotStep.class); - dbCleanerStep = mock(DbCleanerStep.class); + dataCleanerStep = mock(DataCleanerStep.class); - sut = new ComputationStepRegistry(getAndSetProjectStep, synchronizeProjectPermissionsStep, indexProjectIssuesStep, switchSnapshotStep, dbCleanerStep); + sut = new ComputationStepRegistry(getAndSetProjectStep, synchronizeProjectPermissionsStep, indexProjectIssuesStep, switchSnapshotStep, dataCleanerStep); } @Test public void steps_returned_in_the_right_order() throws Exception { - assertThat(sut.steps()).containsExactly(getAndSetProjectStep, synchronizeProjectPermissionsStep, switchSnapshotStep, dbCleanerStep, indexProjectIssuesStep); + assertThat(sut.steps()).containsExactly(getAndSetProjectStep, synchronizeProjectPermissionsStep, switchSnapshotStep, dataCleanerStep, indexProjectIssuesStep); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/DbCleanerStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/DataCleanerStepTest.java similarity index 77% rename from server/sonar-server/src/test/java/org/sonar/server/computation/DbCleanerStepTest.java rename to server/sonar-server/src/test/java/org/sonar/server/computation/DataCleanerStepTest.java index 2c05a1a3045..d2f76c270e3 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/DbCleanerStepTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/DataCleanerStepTest.java @@ -22,23 +22,32 @@ package org.sonar.server.computation; import org.junit.Before; import org.junit.Test; +import org.sonar.api.config.Settings; import org.sonar.core.component.ComponentDto; import org.sonar.core.computation.db.AnalysisReportDto; import org.sonar.core.computation.dbcleaner.DefaultPurgeTask; import org.sonar.core.persistence.DbSession; +import org.sonar.server.issue.index.IssueIndex; + +import java.util.Date; import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; -public class DbCleanerStepTest { +public class DataCleanerStepTest { - private DbCleanerStep sut; + private DataCleanerStep sut; private DefaultPurgeTask purgeTask; + private IssueIndex issueIndex; + private Settings settings; @Before public void before() { this.purgeTask = mock(DefaultPurgeTask.class); - this.sut = new DbCleanerStep(purgeTask); + this.issueIndex = mock(IssueIndex.class); + this.settings = mock(Settings.class); + + this.sut = new DataCleanerStep(purgeTask, issueIndex, settings); } @Test @@ -49,6 +58,6 @@ public class DbCleanerStepTest { sut.execute(mock(DbSession.class), report); verify(purgeTask).purge(any(Long.class)); + verify(issueIndex).deleteClosedIssuesOfProjectBefore(anyString(), any(Date.class)); } - } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/DbCleanStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/DbCleanStepTest.java deleted file mode 100644 index 8e092313b50..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/DbCleanStepTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package org.sonar.server.computation; - -import org.junit.Before; -import org.sonar.api.config.Settings; -import org.sonar.core.computation.dbcleaner.DefaultPurgeTask; -import org.sonar.core.computation.dbcleaner.period.DefaultPeriodCleaner; -import org.sonar.core.purge.PurgeDao; -import org.sonar.core.purge.PurgeProfiler; - -import static org.mockito.Mockito.mock; - -public class DbCleanStepTest { - - private DbCleanerStep sut; - - @Before - public void before() { - sut = new DbCleanerStep(new DefaultPurgeTask(mock(PurgeDao.class), mock(Settings.class), mock(DefaultPeriodCleaner.class), mock(PurgeProfiler.class))); - } - -} \ No newline at end of file diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexMediumTest.java index 9524c78ed13..0139d113aa0 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexMediumTest.java @@ -863,4 +863,31 @@ public class IssueIndexMediumTest { assertThat(searchResult.getHits()).isEmpty(); assertThat(index.countAll()).isEqualTo(1); } + + @Test + public void delete_closed_issues_from_one_project_older_than_specific_date() { + // ARRANGE + Date today = new Date(); + Date yesterday = org.apache.commons.lang.time.DateUtils.addDays(today, -1); + Date beforeYesterday = org.apache.commons.lang.time.DateUtils.addDays(yesterday, -1); + + tester.get(IssueDao.class).insert(session, IssueTesting.newDto(rule, file, project).setIssueCloseDate(today)); + tester.get(IssueDao.class).insert(session, IssueTesting.newDto(rule, file, project).setIssueCloseDate(beforeYesterday)); + tester.get(IssueDao.class).insert(session, IssueTesting.newDto(rule, file, project)); + session.commit(); + assertThat(index.countAll()).isEqualTo(3L); + + // ACT + index.deleteClosedIssuesOfProjectBefore(project.uuid(), yesterday); + + // ASSERT + List issues = index.search(IssueQuery.builder().componentRootUuids(newArrayList(project.uuid())).build(), new QueryContext()).getHits(); + List dates = newArrayList(); + for (Issue issue : issues) { + dates.add(issue.closeDate()); + } + + assertThat(index.countAll()).isEqualTo(2); + assertThat(dates).containsOnly(null, today); + } } diff --git a/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/DefaultPurgeTask.java b/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/DefaultPurgeTask.java index 60e76cf0a8c..63957a0a7f6 100644 --- a/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/DefaultPurgeTask.java +++ b/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/DefaultPurgeTask.java @@ -24,7 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.CoreProperties; import org.sonar.api.config.Settings; -import org.sonar.api.resources.Scopes; import org.sonar.api.utils.TimeUtils; import org.sonar.core.computation.dbcleaner.period.DefaultPeriodCleaner; import org.sonar.core.purge.PurgeConfiguration; @@ -32,15 +31,17 @@ import org.sonar.core.purge.PurgeDao; import org.sonar.core.purge.PurgeProfiler; import org.sonar.plugins.dbcleaner.api.PurgeTask; +import static org.sonar.core.purge.PurgeConfiguration.newDefaultPurgeConfiguration; + /** * @since 2.14 */ public class DefaultPurgeTask implements PurgeTask { private static final Logger LOG = LoggerFactory.getLogger(DefaultPurgeTask.class); private final PurgeProfiler profiler; - private PurgeDao purgeDao; - private Settings settings; - private DefaultPeriodCleaner periodCleaner; + private final PurgeDao purgeDao; + private final Settings settings; + private final DefaultPeriodCleaner periodCleaner; public DefaultPurgeTask(PurgeDao purgeDao, Settings settings, DefaultPeriodCleaner periodCleaner, PurgeProfiler profiler) { this.purgeDao = purgeDao; @@ -81,18 +82,14 @@ public class DefaultPurgeTask implements PurgeTask { private void doPurge(long resourceId) { try { - purgeDao.purge(newConf(resourceId)); + purgeDao.purge(newPurgeConfigurationOnResource(resourceId)); } catch (Exception e) { // purge errors must no fail the batch LOG.error("Fail to purge data [id=" + resourceId + "]", e); } } - private PurgeConfiguration newConf(long resourceId) { - String[] scopes = new String[] {Scopes.FILE}; - if (settings.getBoolean(DbCleanerConstants.PROPERTY_CLEAN_DIRECTORY)) { - scopes = new String[] {Scopes.DIRECTORY, Scopes.FILE}; - } - return new PurgeConfiguration(resourceId, scopes, settings.getInt(DbCleanerConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES)); + public PurgeConfiguration newPurgeConfigurationOnResource(long resourceId) { + return newDefaultPurgeConfiguration(resourceId, settings); } } diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeConfiguration.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeConfiguration.java index d026ba60612..1078be70dbd 100644 --- a/sonar-core/src/main/java/org/sonar/core/purge/PurgeConfiguration.java +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeConfiguration.java @@ -21,7 +21,10 @@ package org.sonar.core.purge; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.time.DateUtils; +import org.sonar.api.config.Settings; +import org.sonar.api.resources.Scopes; import org.sonar.api.utils.System2; +import org.sonar.core.computation.dbcleaner.DbCleanerConstants; import javax.annotation.CheckForNull; @@ -46,6 +49,14 @@ public class PurgeConfiguration { this.system2 = system2; } + public static PurgeConfiguration newDefaultPurgeConfiguration(long resourceId, Settings settings) { + String[] scopes = new String[] {Scopes.FILE}; + if (settings.getBoolean(DbCleanerConstants.PROPERTY_CLEAN_DIRECTORY)) { + scopes = new String[] {Scopes.DIRECTORY, Scopes.FILE}; + } + return new PurgeConfiguration(resourceId, scopes, settings.getInt(DbCleanerConstants.DAYS_BEFORE_DELETING_CLOSED_ISSUES)); + } + public long rootProjectId() { return rootProjectId; } -- 2.39.5