SONAR-7309 DB Cleaner strategy to keep only analyses with a version

server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java

@@ -114,7 +114,7 @@ public class ComputeEngineContainerImplTest {
         + 26 // level 1
         + 52 // content of DaoModule
         + 3 // content of EsSearchModule
-        + 66 // content of CorePropertyDefinitions
+        + 67 // content of CorePropertyDefinitions
         + 1 // StopFlagContainer
     );
     assertThat(

server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeableAnalysisDto.java

@@ -20,16 +20,19 @@
 package org.sonar.db.purge;
 
 import java.util.Date;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
 import org.apache.commons.lang.builder.ReflectionToStringBuilder;
 import org.apache.commons.lang.builder.ToStringStyle;
 
 /**
- * Represents an analysis, aka. root snapshot, aka. snapshot of a project, developer or view
+ * Represents an analysis, aka. root snapshot, aka. snapshot of a project or portfolio
  */
 public class PurgeableAnalysisDto implements Comparable<PurgeableAnalysisDto> {
   private Date date;
   private long analysisId;
   private String analysisUuid;
+  private String version;
   private boolean hasEvents;
   private boolean isLast;
 
@@ -78,6 +81,16 @@ public class PurgeableAnalysisDto implements Comparable<PurgeableAnalysisDto> {
     return this;
   }
 
+  @CheckForNull
+  public String getVersion() {
+    return version;
+  }
+
+  public PurgeableAnalysisDto setVersion(@Nullable String version) {
+    this.version = version;
+    return this;
+  }
+
   @Override
   public int compareTo(PurgeableAnalysisDto other) {
     return date.compareTo(other.date);

server/sonar-db-dao/src/main/java/org/sonar/db/purge/period/Filters.java

@@ -34,11 +34,13 @@ class Filters {
     Date dateToStartKeepingOneSnapshotByDay = getDateFromHours(config, PurgeConstants.HOURS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_DAY);
     Date dateToStartKeepingOneSnapshotByWeek = getDateFromWeeks(config, PurgeConstants.WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_WEEK);
     Date dateToStartKeepingOneSnapshotByMonth = getDateFromWeeks(config, PurgeConstants.WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_MONTH);
+    Date dateToStartKeepingOnlyAnalysisWithVersion = getDateFromWeeks(config, PurgeConstants.WEEKS_BEFORE_KEEPING_ONLY_ANALYSES_WITH_VERSION);
     Date dateToStartDeletingAllSnapshots = getDateFromWeeks(config, PurgeConstants.WEEKS_BEFORE_DELETING_ALL_SNAPSHOTS);
 
     all.add(new KeepOneFilter(dateToStartKeepingOneSnapshotByWeek, dateToStartKeepingOneSnapshotByDay, Calendar.DAY_OF_YEAR, "day"));
     all.add(new KeepOneFilter(dateToStartKeepingOneSnapshotByMonth, dateToStartKeepingOneSnapshotByWeek, Calendar.WEEK_OF_YEAR, "week"));
     all.add(new KeepOneFilter(dateToStartDeletingAllSnapshots, dateToStartKeepingOneSnapshotByMonth, Calendar.MONTH, "month"));
+    all.add(new KeepWithVersionFilter(dateToStartKeepingOnlyAnalysisWithVersion));
     all.add(new DeleteAllFilter(dateToStartDeletingAllSnapshots));
   }
 

server/sonar-db-dao/src/main/java/org/sonar/db/purge/period/KeepOneFilter.java

@@ -57,7 +57,7 @@ class KeepOneFilter implements Filter {
     Loggers.get(getClass()).debug("-> Keep one snapshot per {} between {} and {}", label, DateUtils.formatDate(start), DateUtils.formatDate(end));
   }
 
-  private void appendSnapshotsToDelete(Interval interval, List<PurgeableAnalysisDto> toDelete) {
+  private static void appendSnapshotsToDelete(Interval interval, List<PurgeableAnalysisDto> toDelete) {
     if (interval.count() > 1) {
       List<PurgeableAnalysisDto> deletables = Lists.newArrayList();
       List<PurgeableAnalysisDto> toKeep = Lists.newArrayList();

server/sonar-db-dao/src/main/java/org/sonar/db/purge/period/KeepWithVersionFilter.java

@@ -0,0 +1,55 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.purge.period;
+
+import com.google.common.base.Strings;
+import java.util.Date;
+import java.util.List;
+import org.sonar.api.utils.DateUtils;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.core.util.stream.MoreCollectors;
+import org.sonar.db.purge.PurgeableAnalysisDto;
+
+class KeepWithVersionFilter implements Filter {
+
+  private final Date before;
+
+  KeepWithVersionFilter(Date before) {
+    this.before = before;
+  }
+
+  @Override
+  public List<PurgeableAnalysisDto> filter(List<PurgeableAnalysisDto> history) {
+    return history.stream()
+      .filter(analysis -> analysis.getDate().before(before))
+      .filter(KeepWithVersionFilter::isDeletable)
+      .collect(MoreCollectors.toList());
+  }
+
+  @Override
+  public void log() {
+    Loggers.get(getClass()).debug("-> Keep analyses with a version prior to {}", DateUtils.formatDate(before));
+  }
+
+  private static boolean isDeletable(PurgeableAnalysisDto snapshot) {
+    return !snapshot.isLast() && Strings.isNullOrEmpty(snapshot.getVersion());
+  }
+
+}

server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml

@@ -27,9 +27,9 @@
 
   <select id="selectPurgeableAnalysesWithEvents" parameterType="String" resultType="PurgeableAnalysis">
     select
-      s.id as "analysisId", s.uuid as "analysisUuid", s.created_at as "date", ${_true} as "hasEvents", islast as "isLast"
-    from
-      snapshots s
+      s.id as "analysisId", s.uuid as "analysisUuid", s.created_at as "date", ${_true} as "hasEvents", islast as "isLast", ve.name as "version"
+    from snapshots s
+      left outer join events ve on ve.analysis_uuid=s.uuid and ve.category='Version'
     where
       s.component_uuid=#{componentUuid,jdbcType=VARCHAR}
       and s.status='P'
@@ -38,9 +38,8 @@
 
   <select id="selectPurgeableAnalysesWithoutEvents" parameterType="String" resultType="PurgeableAnalysis">
     select
-      s.id as "analysisId", s.uuid as "analysisUuid", s.created_at as "date", ${_false} as "hasEvents", islast as "isLast"
-    from
-      snapshots s
+      s.id as "analysisId", s.uuid as "analysisUuid", s.created_at as "date", ${_false} as "hasEvents", islast as "isLast", NULL as "version"
+    from snapshots s
     where
       s.component_uuid=#{componentUuid,jdbcType=VARCHAR}
       and s.status='P'

server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java

@@ -161,17 +161,20 @@ public class PurgeDaoTest {
   }
 
   @Test
-  public void shouldSelectPurgeableAnalysis() {
+  public void selectPurgeableAnalyses() {
     dbTester.prepareDbUnit(getClass(), "shouldSelectPurgeableAnalysis.xml");
     List<PurgeableAnalysisDto> analyses = underTest.selectPurgeableAnalyses(THE_PROJECT_UUID, dbSession);
 
     assertThat(analyses).hasSize(3);
     assertThat(getById(analyses, "u1").isLast()).isTrue();
     assertThat(getById(analyses, "u1").hasEvents()).isFalse();
+    assertThat(getById(analyses, "u1").getVersion()).isNull();
     assertThat(getById(analyses, "u4").isLast()).isFalse();
     assertThat(getById(analyses, "u4").hasEvents()).isFalse();
+    assertThat(getById(analyses, "u4").getVersion()).isNull();
     assertThat(getById(analyses, "u5").isLast()).isFalse();
     assertThat(getById(analyses, "u5").hasEvents()).isTrue();
+    assertThat(getById(analyses, "u5").getVersion()).isEqualTo("V5");
   }
 
   @Test

server/sonar-db-dao/src/test/java/org/sonar/db/purge/period/KeepWithVersionFilterTest.java

@@ -0,0 +1,44 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.purge.period;
+
+import java.util.Arrays;
+import java.util.List;
+import org.junit.Test;
+import org.sonar.db.purge.DbCleanerTestUtils;
+import org.sonar.db.purge.PurgeableAnalysisDto;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.sonar.api.utils.DateUtils.parseDate;
+
+public class KeepWithVersionFilterTest {
+
+  @Test
+  public void keep_only_analyses_with_a_version() {
+    Filter underTest = new KeepWithVersionFilter(parseDate("2015-10-18"));
+
+    List<PurgeableAnalysisDto> result = underTest.filter(Arrays.asList(
+      DbCleanerTestUtils.createAnalysisWithDate("u1", "2015-10-17").setVersion("V1"),
+      DbCleanerTestUtils.createAnalysisWithDate("u2", "2015-10-17").setVersion(null),
+      DbCleanerTestUtils.createAnalysisWithDate("u3", "2015-10-19").setVersion(null)));
+
+    assertThat(result).extracting(PurgeableAnalysisDto::getAnalysisUuid).containsExactlyInAnyOrder("u2");
+  }
+}

server/sonar-db-dao/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldSelectPurgeableAnalysis.xml

@@ -132,7 +132,7 @@
              period5_date="[null]"
              created_at="1228222680000"
              build_date="1228222680000"
-             version="[null]"
+             version="V5"
   />
 
   <events id="2"
@@ -141,7 +141,7 @@
           component_uuid="1"
           category="Version"
           description="[null]"
-          name="Version 1.0"
+          name="V5"
           event_date="1228222680000"
           created_at="1228222680000"
           event_data="[null]"/>

sonar-core/src/main/java/org/sonar/core/config/PurgeConstants.java

@@ -25,6 +25,7 @@ public interface PurgeConstants {
   String HOURS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_DAY = "sonar.dbcleaner.hoursBeforeKeepingOnlyOneSnapshotByDay";
   String WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_WEEK = "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByWeek";
   String WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_MONTH = "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByMonth";
+  String WEEKS_BEFORE_KEEPING_ONLY_ANALYSES_WITH_VERSION = "sonar.dbcleaner.weeksBeforeKeepingOnlyAnalysesWithVersion";
   String WEEKS_BEFORE_DELETING_ALL_SNAPSHOTS = "sonar.dbcleaner.weeksBeforeDeletingAllSnapshots";
   String DAYS_BEFORE_DELETING_CLOSED_ISSUES = "sonar.dbcleaner.daysBeforeDeletingClosedIssues";
   String DAYS_BEFORE_DELETING_INACTIVE_SHORT_LIVING_BRANCHES = "sonar.dbcleaner.daysBeforeDeletingInactiveShortLivingBranches";

sonar-core/src/main/java/org/sonar/core/config/PurgeProperties.java

@@ -101,6 +101,17 @@ public final class PurgeProperties {
         .category(CoreProperties.CATEGORY_GENERAL)
         .subCategory(CoreProperties.SUBCATEGORY_DATABASE_CLEANER)
         .index(6)
+        .build(),
+
+      PropertyDefinition.builder(PurgeConstants.WEEKS_BEFORE_KEEPING_ONLY_ANALYSES_WITH_VERSION)
+        .defaultValue("104")
+        .name("Keep only analyses with a version event after")
+        .description("After this number of weeks, the DbCleaner keeps only analyses with a version event associated.")
+        .type(PropertyType.INTEGER)
+        .onQualifiers(Qualifiers.PROJECT)
+        .category(CoreProperties.CATEGORY_GENERAL)
+        .subCategory(CoreProperties.SUBCATEGORY_DATABASE_CLEANER)
+        .index(7)
         .build()
       );
   }

sonar-core/src/test/java/org/sonar/core/config/CorePropertyDefinitionsTest.java

@@ -33,7 +33,7 @@ public class CorePropertyDefinitionsTest {
   @Test
   public void all() {
     List<PropertyDefinition> defs = CorePropertyDefinitions.all();
-    assertThat(defs).hasSize(66);
+    assertThat(defs).hasSize(67);
   }
 
   @Test

sonar-core/src/test/java/org/sonar/core/config/PurgePropertiesTest.java

@@ -27,6 +27,6 @@ public class PurgePropertiesTest {
 
   @Test
   public void shouldGetExtensions() {
-    assertThat(PurgeProperties.all()).hasSize(6);
+    assertThat(PurgeProperties.all()).hasSize(7);
   }
 }

tests/src/test/java/org/sonarqube/tests/dbCleaner/PurgeTest.java

@@ -22,7 +22,6 @@ package org.sonarqube.tests.dbCleaner;
 import com.sonar.orchestrator.Orchestrator;
 import com.sonar.orchestrator.build.BuildResult;
 import com.sonar.orchestrator.build.SonarScanner;
-import org.sonarqube.tests.Category4Suite;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
@@ -35,9 +34,13 @@ import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ErrorCollector;
+import org.sonarqube.tests.Category4Suite;
+import org.sonarqube.tests.Tester;
+import org.sonarqube.ws.ProjectAnalyses;
 import org.sonarqube.ws.WsMeasures;
 import org.sonarqube.ws.WsMeasures.SearchHistoryResponse.HistoryValue;
 import org.sonarqube.ws.client.measure.SearchHistoryRequest;
+import org.sonarqube.ws.client.projectanalysis.SearchRequest;
 import util.ItUtils;
 
 import static java.util.Collections.singletonList;
@@ -46,8 +49,10 @@ import static org.assertj.core.api.Assertions.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 import static util.ItUtils.formatDate;
 import static util.ItUtils.newAdminWsClient;
+import static util.ItUtils.restoreProfile;
 import static util.ItUtils.runProjectAnalysis;
 import static util.ItUtils.setServerProperty;
+import static util.ItUtils.toDatetime;
 
 @Ignore
 public class PurgeTest {
@@ -62,7 +67,8 @@ public class PurgeTest {
 
   @ClassRule
   public static final Orchestrator orchestrator = Category4Suite.ORCHESTRATOR;
-
+  @Rule
+  public Tester tester = new Tester(orchestrator);
   @Rule
   public ErrorCollector collector = new ErrorCollector();
 
@@ -72,11 +78,7 @@ public class PurgeTest {
 
     orchestrator.getServer().provisionProject(PROJECT_KEY, PROJECT_KEY);
 
-    ItUtils.restoreProfile(orchestrator, getClass().getResource("/dbCleaner/one-issue-per-line-profile.xml"));
-
-    setServerProperty(orchestrator, "sonar.dbcleaner.cleanDirectory", null);
-    setServerProperty(orchestrator, "sonar.dbcleaner.hoursBeforeKeepingOnlyOneSnapshotByDay", null);
-    setServerProperty(orchestrator, "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByWeek", null);
+    restoreProfile(orchestrator, getClass().getResource("/dbCleaner/one-issue-per-line-profile.xml"));
   }
 
   @Test
@@ -216,6 +218,25 @@ public class PurgeTest {
     assertThat(response.getMeasuresList().get(0).getHistoryList()).extracting(HistoryValue::getDate).doesNotContain(lastWednesdayFormatted, lastThursdayFormatted);
   }
 
+  @Test
+  public void keep_only_analyses_with_a_version() {
+    tester.settings().setGlobalSettings("sonar.dbcleaner.weeksBeforeKeepingOnlyAnalysesWithVersion", "1");
+
+    Date now = new Date();
+    String heightDaysAgo = formatDate(addDays(now, -8));
+    String nineDaysAgo = formatDate(addDays(now, -9));
+
+    runProjectAnalysis(orchestrator, PROJECT_SAMPLE_PATH, "sonar.projectDate", nineDaysAgo, "sonar.projectVersion", "V1");
+    runProjectAnalysis(orchestrator, PROJECT_SAMPLE_PATH, "sonar.projectDate", heightDaysAgo);
+    runProjectAnalysis(orchestrator, PROJECT_SAMPLE_PATH);
+
+    ProjectAnalyses.SearchResponse result = tester.wsClient().projectAnalysis().search(SearchRequest.builder().setProject(PROJECT_KEY).build());
+
+    assertThat(result.getAnalysesList()).extracting(a -> formatDate(toDatetime(a.getDate())))
+      .contains(nineDaysAgo, formatDate(now))
+      .doesNotContain(heightDaysAgo);
+  }
+
   /**
    * SONAR-3120
    */

tests/src/test/java/org/sonarqube/tests/measure/TimeMachineTest.java

@@ -22,7 +22,6 @@ package org.sonarqube.tests.measure;
 import com.sonar.orchestrator.Orchestrator;
 import com.sonar.orchestrator.build.BuildResult;
 import com.sonar.orchestrator.build.SonarScanner;
-import org.sonarqube.tests.Category1Suite;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.Map;
@@ -30,6 +29,7 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.Test;
+import org.sonarqube.tests.Category1Suite;
 import org.sonarqube.ws.WsMeasures.Measure;
 import org.sonarqube.ws.WsMeasures.SearchHistoryResponse;
 import org.sonarqube.ws.WsMeasures.SearchHistoryResponse.HistoryValue;
@@ -39,6 +39,7 @@ import util.ItUtils;
 import util.ItUtils.ComponentNavigation;
 
 import static java.util.Collections.singletonList;
+import static org.apache.commons.lang.time.DateUtils.addDays;
 import static org.assertj.core.api.Assertions.assertThat;
 import static util.ItUtils.formatDate;
 import static util.ItUtils.getComponentNavigation;
@@ -51,8 +52,6 @@ import static util.ItUtils.setServerProperty;
 public class TimeMachineTest {
 
   private static final String PROJECT = "sample";
-  private static final String FIRST_ANALYSIS_DATE = "2014-10-19";
-  private static final String SECOND_ANALYSIS_DATE = "2014-11-13";
 
   @ClassRule
   public static Orchestrator orchestrator = Category1Suite.ORCHESTRATOR;
@@ -65,8 +64,12 @@ public class TimeMachineTest {
     ItUtils.restoreProfile(orchestrator, TimeMachineTest.class.getResource("/measure/one-issue-per-line-profile.xml"));
     orchestrator.getServer().provisionProject("sample", "Sample");
     orchestrator.getServer().associateProjectToQualityProfile("sample", "xoo", "one-issue-per-line");
-    analyzeProject("measure/xoo-history-v1", FIRST_ANALYSIS_DATE);
-    analyzeProject("measure/xoo-history-v2", SECOND_ANALYSIS_DATE);
+
+    Date now = new Date();
+    String yesterday = formatDate(addDays(now, -1));
+    String aMonthAgo = formatDate(addDays(now, -30));
+    analyzeProject("measure/xoo-history-v1", aMonthAgo);
+    analyzeProject("measure/xoo-history-v2", yesterday);
 
     wsMeasures = newAdminWsClient(orchestrator).measures();
   }
@@ -88,7 +91,6 @@ public class TimeMachineTest {
   public void projectIsAnalyzed() {
     ComponentNavigation component = getComponentNavigation(orchestrator, PROJECT);
     assertThat(component.getVersion()).isEqualTo("1.0-SNAPSHOT");
-    assertThat(component.getDate().getMonth()).isEqualTo(10); // November
   }
 
   @Test

tests/src/test/java/util/ItUtils.java

@@ -477,6 +477,11 @@ public class ItUtils {
     return sdf.format(d);
   }
 
+  public static String formatDateTime(Date d) {
+    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
+    return sdf.format(d);
+  }
+
   public static String extractCeTaskId(BuildResult buildResult) {
     List<String> taskIds = extractCeTaskIds(buildResult);
     checkState(taskIds.size() == 1, "More than one task id retrieved from logs");