SONAR-12691 Don't persist project measures that are not historical

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/measure/Measure.java

    */
   double getVariation();
 
+  default boolean isEmpty() {
+    return getValueType() == ValueType.NO_VALUE && !hasVariation() && getData() == null;
+  }
+
   static NewMeasureBuilder newMeasureBuilder() {
     return new NewMeasureBuilder();
   }

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/metric/Metric.java

    */
   int getDecimalScale();
 
+  boolean isDeleteHistoricalData();
+
   enum MetricType {
     INT(Measure.ValueType.INT),
     MILLISEC(Measure.ValueType.LONG),

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/metric/MetricDtoToMetric.java

     return new MetricImpl(
       metricDto.getId(), metricDto.getKey(), metricDto.getShortName(), metricType,
       decimalScale,
-      DoubleCache.intern(metricDto.getBestValue()), metricDto.isOptimizedBestValue());
+      DoubleCache.intern(metricDto.getBestValue()), metricDto.isOptimizedBestValue(),
+      metricDto.isDeleteHistoricalData());
   }
 }

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/metric/MetricImpl.java

   private final Integer decimalScale;
   private final Double bestValue;
   private final boolean bestValueOptimized;
+  private boolean deleteHistoricalData;
 
   public MetricImpl(int id, String key, String name, MetricType type) {
-    this(id, key, name, type, null, null, false);
+    this(id, key, name, type, null, null, false, false);
   }
 
   public MetricImpl(int id, String key, String name, MetricType type, @Nullable Integer decimalScale,
-    @Nullable Double bestValue, boolean bestValueOptimized) {
+    @Nullable Double bestValue, boolean bestValueOptimized, boolean deleteHistoricalData) {
     checkArgument(!bestValueOptimized || bestValue != null, "A BestValue must be specified if Metric is bestValueOptimized");
     this.id = id;
     this.key = checkNotNull(key);
     }
     this.bestValueOptimized = bestValueOptimized;
     this.bestValue = bestValue;
+    this.deleteHistoricalData = deleteHistoricalData;
   }
 
   @Override
     return bestValueOptimized;
   }
 
+  @Override
+  public boolean isDeleteHistoricalData() {
+    return deleteHistoricalData;
+  }
+
   @Override
   public boolean equals(@Nullable Object o) {
     if (this == o) {
       .add("type", type)
       .add("bestValue", bestValue)
       .add("bestValueOptimized", bestValueOptimized)
+      .add("deleteHistoricalData", deleteHistoricalData)
       .toString();
   }
 }

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistMeasuresStep.java

 package org.sonar.ce.task.projectanalysis.step;
 
 import java.util.Map;
-import java.util.function.Predicate;
-import javax.annotation.Nonnull;
 import org.sonar.ce.task.projectanalysis.component.Component;
 import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit;
 import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler;
       persistMeasures(project);
     }
 
-    @Override
-    public void visitDirectory(Component directory) {
-      // measures of directories are never read. No need to persist them.
-    }
-
     @Override
     public void visitView(Component view) {
       persistMeasures(view);
 
       for (Map.Entry<String, Measure> e : measures.entrySet()) {
         Measure measure = e.getValue();
-        if (!NonEmptyMeasure.INSTANCE.test(measure)) {
+        if (measure.isEmpty()) {
           continue;
         }
         String metricKey = e.getKey();
         Metric metric = metricRepository.getByKey(metricKey);
-        MeasureDto measureDto = measureToMeasureDto.toMeasureDto(measure, metric, component);
-        measureDao.insert(session, measureDto);
-        inserts++;
+        if (!metric.isDeleteHistoricalData()) {
+          MeasureDto measureDto = measureToMeasureDto.toMeasureDto(measure, metric, component);
+          measureDao.insert(session, measureDto);
+          inserts++;
+        }
       }
     }
-
-  }
-
-
-  private enum NonEmptyMeasure implements Predicate<Measure> {
-    INSTANCE;
-
-    @Override
-    public boolean test(@Nonnull Measure input) {
-      return input.getValueType() != Measure.ValueType.NO_VALUE || input.hasVariation() || input.getData() != null;
-    }
   }
-
 }

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/BestValueOptimizationTest.java

   }
 
   private static MetricImpl createMetric(Metric.MetricType metricType, double bestValue) {
-    return new MetricImpl(metricType.hashCode() + (int) bestValue, "key" + metricType + bestValue, "name" + metricType + bestValue, metricType, null, bestValue, true);
+    return new MetricImpl(metricType.hashCode() + (int) bestValue, "key" + metricType + bestValue, "name" + metricType + bestValue, metricType, null,
+      bestValue, true, false);
   }
 }

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/LiveMeasureDtoToMeasureTest.java

 
   @Test
   public void toMeasure_should_not_loose_decimals_of_float_values() {
-    MetricImpl metric = new MetricImpl(42, "double", "name", Metric.MetricType.FLOAT, 5, null, false);
+    MetricImpl metric = new MetricImpl(42, "double", "name", Metric.MetricType.FLOAT, 5, null, false, false);
     LiveMeasureDto LiveMeasureDto = new LiveMeasureDto()
       .setValue(0.12345);
 

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/MeasureDtoToMeasureTest.java

 
   @Test
   public void toMeasure_should_not_loose_decimals_of_float_values() {
-    MetricImpl metric = new MetricImpl(42, "double", "name", Metric.MetricType.FLOAT, 5, null, false);
+    MetricImpl metric = new MetricImpl(42, "double", "name", Metric.MetricType.FLOAT, 5, null, false, false);
     MeasureDto measureDto = new MeasureDto()
       .setValue(0.12345);
 

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/metric/MetricDtoToMetricTest.java

       assertThat(metric.getType()).isEqualTo(metricType);
       assertThat(metric.isBestValueOptimized()).isFalse();
       assertThat(metric.getBestValue()).isEqualTo(SOME_BEST_VALUE);
+      assertThat(metric.isDeleteHistoricalData()).isTrue();
+
     }
   }
 
       .setShortName(metricType.name() + "_name")
       .setValueType(metricType.name())
       .setBestValue(SOME_BEST_VALUE)
+      .setDeleteHistoricalData(true)
       .setEnabled(true);
   }
 }

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/metric/MetricImplTest.java

     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage("A BestValue must be specified if Metric is bestValueOptimized");
 
-    new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.INT, 1, null, true);
+    new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.INT, 1, null, true, false);
   }
 
   @Test
 
     assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.FLOAT)).isEqualTo(expected);
     assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING)).isEqualTo(expected);
-    assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING, null, 0d, true)).isEqualTo(expected);
-    assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING, null, null, false)).isEqualTo(expected);
+    assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING, null, 0d, true, true)).isEqualTo(expected);
+    assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING, null, null, false, false)).isEqualTo(expected);
     assertThat(new MetricImpl(SOME_ID, "some other key", SOME_NAME, Metric.MetricType.FLOAT)).isNotEqualTo(expected);
   }
 
 
   @Test
   public void all_fields_are_displayed_in_toString() {
-    assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.FLOAT, 1, 951d, true).toString())
-      .isEqualTo("MetricImpl{id=42, key=key, name=name, type=FLOAT, bestValue=951.0, bestValueOptimized=true}");
+    assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.FLOAT, 1, 951d, true, false).toString())
+      .isEqualTo("MetricImpl{id=42, key=key, name=name, type=FLOAT, bestValue=951.0, bestValueOptimized=true, deleteHistoricalData=false}");
 
   }
 }

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistMeasuresStepTest.java

 
   private static final Metric STRING_METRIC = new Metric.Builder("string-metric", "String metric", Metric.ValueType.STRING).create();
   private static final Metric INT_METRIC = new Metric.Builder("int-metric", "int metric", Metric.ValueType.INT).create();
+  private static final Metric NON_HISTORICAL_METRIC = new Metric.Builder("nh-metric", "nh metric", Metric.ValueType.INT).setDeleteHistoricalData(true).create();
 
   private static final String ANALYSIS_UUID = "a1";
 
     analysisMetadataHolder.setUuid(ANALYSIS_UUID);
     MetricDto stringMetricDto = db.measures().insertMetric(m -> m.setKey(STRING_METRIC.getKey()).setValueType(Metric.ValueType.STRING.name()));
     MetricDto intMetricDto = db.measures().insertMetric(m -> m.setKey(INT_METRIC.getKey()).setValueType(Metric.ValueType.INT.name()));
+    MetricDto nhMetricDto = db.measures().insertMetric(m -> m.setKey(NON_HISTORICAL_METRIC.getKey()).setValueType(Metric.ValueType.INT.name()));
     metricRepository.add(stringMetricDto.getId(), STRING_METRIC);
     metricRepository.add(intMetricDto.getId(), INT_METRIC);
+    metricRepository.add(nhMetricDto.getId(), NON_HISTORICAL_METRIC);
+  }
+
+  @Test
+  public void measures_on_non_historical_metrics_are_not_persisted() {
+    prepareProject();
+    measureRepository.addRawMeasure(REF_1, NON_HISTORICAL_METRIC.getKey(), newMeasureBuilder().create(1));
+    measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().create(2));
+
+    TestComputationStepContext context = execute();
+
+    assertThatMeasureIsNotPersisted("project-uuid", NON_HISTORICAL_METRIC);
+    MeasureDto persistedMeasure = selectMeasure("project-uuid", INT_METRIC).get();
+    assertThat(persistedMeasure.getValue()).isEqualTo(2);
+    assertNbOfInserts(context, 1);
   }
 
   @Test

server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/metric/MetricRepositoryRule.java

       id, coreMetric.getKey(), coreMetric.getName(),
       convert(coreMetric.getType()),
       coreMetric.getDecimalScale(),
-      coreMetric.getBestValue(), coreMetric.isOptimizedBestValue());
+      coreMetric.getBestValue(), coreMetric.isOptimizedBestValue(), coreMetric.getDeleteHistoricalData());
   }
 
   private static Metric.MetricType convert(org.sonar.api.measures.Metric.ValueType coreMetricType) {

server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java

     return mapper(dbSession).selectByQueryOnSingleComponent(query);
   }
 
-  public List<PastMeasureDto> selectPastMeasures(DbSession dbSession, String componentUuid, String analysisUuid, Collection<Integer> metricIds) {
-    if (metricIds.isEmpty()) {
-      return emptyList();
-    }
-    return executeLargeInputs(
-      metricIds,
-      ids -> mapper(dbSession).selectPastMeasuresOnSingleAnalysis(componentUuid, analysisUuid, ids));
-  }
-
   /**
    * Select measures of:
    * - one component

server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java

 
   List<MeasureDto> selectByQueryOnSingleComponent(@Param("query") MeasureQuery query);
 
-  List<PastMeasureDto> selectPastMeasuresOnSingleAnalysis(@Param("componentUuid") String componentUuid, @Param("analysisUuid") String analysisUuid,
-    @Param("metricIds") List<Integer> metricIds);
-
   List<MeasureDto> selectPastMeasuresOnSeveralAnalyses(@Param("query") PastMeasureQuery query);
 
   void insert(MeasureDto measureDto);

server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureQuery.java

     this.metricKeys = metricKeys;
   }
 
+  @CheckForNull
   public String getAnalysisUuid() {
     return analysisUuid;
   }

server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java

 
     deleteAnalysisDuplications(analysisUuidsPartitions);
 
-    profiler.start("deleteSnapshotWastedMeasures (project_measures)");
-    List<Long> metricIdsWithoutHistoricalData = purgeMapper.selectMetricIdsWithoutHistoricalData();
-    if (!metricIdsWithoutHistoricalData.isEmpty()) {
-      analysisUuidsPartitions
-        .forEach(analysisUuidsPartition -> purgeMapper.deleteAnalysisWastedMeasures(analysisUuidsPartition, metricIdsWithoutHistoricalData));
-      session.commit();
-    }
-    profiler.stop();
-
     profiler.start("updatePurgeStatusToOne (snapshots)");
     analysisUuidsPartitions.forEach(purgeMapper::updatePurgeStatusToOne);
     session.commit();

server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java

 
   void fullDeleteComponentMeasures(@Param("componentUuids") List<String> componentUuids);
 
-  List<Long> selectMetricIdsWithoutHistoricalData();
-
-  void deleteAnalysisWastedMeasures(@Param("analysisUuids") List<String> analysisUuids, @Param("metricIds") List<Long> metricIds);
-
   /**
    * Purge status flag is used to not attempt to remove duplications & historical data of analyses
    * for which we already removed them.

server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml

     </if>
   </sql>
 
-  <select id="selectPastMeasuresOnSingleAnalysis" parameterType="map" resultType="org.sonar.db.measure.PastMeasureDto">
-    select pm.id as id, pm.metric_id as metricId, pm.value as value
-    from project_measures pm
-    inner join snapshots analysis on analysis.uuid = pm.analysis_uuid
-    where
-    pm.component_uuid = #{componentUuid,jdbcType=VARCHAR}
-    and analysis.uuid = #{analysisUuid,jdbcType=VARCHAR}
-    and pm.metric_id in <foreach item="metricId" collection="metricIds" open="(" separator="," close=")">#{metricId}</foreach>
-  </select>
-
   <select id="selectPastMeasuresOnSeveralAnalyses" parameterType="map" resultType="Measure">
     select <include refid="measureColumns"/>
     from project_measures pm

server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml

         and pb.updated_at < #{toDate}
   </select>
 
-  <select id="selectMetricIdsWithoutHistoricalData" resultType="long">
-    select id from metrics where delete_historical_data=${_true}
-  </select>
-
   <select id="selectRootAndModulesOrSubviewsByProjectUuid" resultType="IdUuidPair" parameterType="String">
     select
       p.id, p.uuid
       </foreach>
   </delete>
 
-  <delete id="deleteAnalysisWastedMeasures" parameterType="map">
-    delete from project_measures
-    <where>
-      analysis_uuid in
-      <foreach collection="analysisUuids" open="(" close=")" item="analysisUuid" separator=",">
-        #{analysisUuid}
-      </foreach>
-      and metric_id in
-      <foreach collection="metricIds" open="(" item="metricId" separator="," close=")">
-        #{metricId,jdbcType=INTEGER}
-      </foreach>
-    </where>
-  </delete>
-
   <update id="updatePurgeStatusToOne" parameterType="map">
     update
       snapshots

server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeCommandsTest.java

     assertThat(countDuplications(analysis4)).isZero();
   }
 
-  @Test
-  public void purgeAnalyses_deletes_measures_of_metrics_without_historical_data() {
-    MetricDto noHistoryMetric = dbTester.measures().insertMetric(t -> t.setDeleteHistoricalData(true));
-    MetricDto withHistoryMetric = dbTester.measures().insertMetric(t -> t.setDeleteHistoricalData(false));
-    ComponentDto project = dbTester.components().insertPrivateProject();
-    SnapshotDto analysis1 = dbTester.components().insertSnapshot(project);
-    SnapshotDto analysis2 = dbTester.components().insertSnapshot(project);
-    SnapshotDto analysis3 = dbTester.components().insertSnapshot(project);
-    SnapshotDto analysis4 = dbTester.components().insertSnapshot(project);
-    int count = 1 + random.nextInt(12);
-    for (SnapshotDto analysis : Arrays.asList(analysis1, analysis2, analysis3, analysis4)) {
-      IntStream.range(0, count).forEach(i -> {
-        dbTester.measures().insertMeasure(project, analysis, noHistoryMetric);
-        dbTester.measures().insertMeasure(project, analysis, withHistoryMetric);
-      });
-    }
-
-    underTest.purgeAnalyses(toIdUuidPairs(analysis1));
-    assertThat(countMeasures(analysis1, noHistoryMetric)).isZero();
-    assertThat(countMeasures(analysis1, withHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis2, noHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis2, withHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis3, noHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis3, withHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis4, noHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis4, withHistoryMetric)).isEqualTo(count);
-
-    underTest.purgeAnalyses(toIdUuidPairs(analysis1, analysis3, analysis4));
-    assertThat(countMeasures(analysis1, withHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis2, noHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis2, noHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis2, withHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis3, noHistoryMetric)).isZero();
-    assertThat(countMeasures(analysis3, withHistoryMetric)).isEqualTo(count);
-    assertThat(countMeasures(analysis4, noHistoryMetric)).isZero();
-    assertThat(countMeasures(analysis4, withHistoryMetric)).isEqualTo(count);
-  }
-
   /**
    * Test that SQL queries execution do not fail with a huge number of parameter
    */

server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java

 import static java.util.Collections.singletonList;
 import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.tuple;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
     assertThat(uuidsIn("projects")).containsOnly(project.uuid());
   }
 
-  @Test
-  public void shouldDeleteNonHistoricalData() {
-    MetricDto metricWithHistory = db.measures().insertMetric(t -> t.setDeleteHistoricalData(false));
-    MetricDto metricWithoutHistory = db.measures().insertMetric(t -> t.setDeleteHistoricalData(true));
-    ComponentDto project = db.components().insertPrivateProject();
-    SnapshotDto lastAnalysis = db.components().insertSnapshot(project, t -> t.setLast(true));
-    SnapshotDto oldAnalysis = db.components().insertSnapshot(project, t -> t.setLast(false));
-    db.measures().insertMeasure(project, lastAnalysis, metricWithHistory);
-    db.measures().insertMeasure(project, lastAnalysis, metricWithoutHistory);
-    db.measures().insertMeasure(project, oldAnalysis, metricWithHistory);
-    db.measures().insertMeasure(project, oldAnalysis, metricWithoutHistory);
-    ComponentDto otherProject = db.components().insertPrivateProject();
-    SnapshotDto otherLastAnalysis = db.components().insertSnapshot(otherProject, t -> t.setLast(true));
-    SnapshotDto otherOldAnalysis = db.components().insertSnapshot(otherProject, t -> t.setLast(false));
-    db.measures().insertMeasure(otherProject, otherLastAnalysis, metricWithHistory);
-    db.measures().insertMeasure(otherProject, otherLastAnalysis, metricWithoutHistory);
-    db.measures().insertMeasure(otherProject, otherOldAnalysis, metricWithHistory);
-    db.measures().insertMeasure(otherProject, otherOldAnalysis, metricWithoutHistory);
-
-    PurgeConfiguration conf = new PurgeConfiguration(project.uuid(), project.uuid(), 30, Optional.of(30), System2.INSTANCE, emptySet());
-
-    underTest.purge(dbSession, conf, PurgeListener.EMPTY, new PurgeProfiler());
-    dbSession.commit();
-
-    assertThat(db.select("select metric_id as \"METRIC\",analysis_uuid as \"ANALYSIS\" from project_measures"))
-      .extracting(t -> ((Long) t.get("METRIC")).intValue(), t -> t.get("ANALYSIS"))
-      .containsOnly(
-        tuple(metricWithHistory.getId(), lastAnalysis.getUuid()),
-        tuple(metricWithoutHistory.getId(), lastAnalysis.getUuid()),
-        tuple(metricWithHistory.getId(), oldAnalysis.getUuid()),
-
-        tuple(metricWithHistory.getId(), otherLastAnalysis.getUuid()),
-        tuple(metricWithoutHistory.getId(), otherLastAnalysis.getUuid()),
-        tuple(metricWithHistory.getId(), otherOldAnalysis.getUuid()),
-        tuple(metricWithoutHistory.getId(), otherOldAnalysis.getUuid()));
-  }
-
   @Test
   public void close_issues_clean_index_and_file_sources_of_disabled_components_specified_by_uuid_in_configuration() {
     RuleDefinitionDto rule = db.rules().insert();