SONARCLOUD-161 upsert live measures on postgresql

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java

@@ -84,24 +84,25 @@ public class PersistLiveMeasuresStep implements ComputationStep {
 
   @Override
   public void execute(ComputationStep.Context context) {
-    try (DbSession dbSession = dbClient.openSession(false)) {
+    boolean supportUpsert = dbClient.getDatabase().getDialect().supportsUpsert();
+    try (DbSession dbSession = dbClient.openSession(supportUpsert)) {
       Component root = treeRootHolder.getRoot();
-      MeasureVisitor visitor = new MeasureVisitor(dbSession);
+      MeasureVisitor visitor = new MeasureVisitor(dbSession, supportUpsert);
       new DepthTraversalTypeAwareCrawler(visitor).visit(root);
       dbSession.commit();
 
       context.getStatistics().add("insertsOrUpdates", visitor.insertsOrUpdates);
-      context.getStatistics().add("deletes", visitor.deleted);
     }
   }
 
   private class MeasureVisitor extends TypeAwareVisitorAdapter {
     private final DbSession dbSession;
+    private final boolean supportUpsert;
     private int insertsOrUpdates = 0;
-    private int deleted = 0;
 
-    private MeasureVisitor(DbSession dbSession) {
+    private MeasureVisitor(DbSession dbSession, boolean supportUpsert) {
       super(CrawlerDepthLimit.LEAVES, PRE_ORDER);
+      this.supportUpsert = supportUpsert;
       this.dbSession = dbSession;
     }
 
@@ -124,12 +125,19 @@ public class PersistLiveMeasuresStep implements ComputationStep {
           // To prevent deadlock, live measures are ordered the same way as in LiveMeasureComputerImpl#refreshComponentsOnSameProject
           .sorted(LiveMeasureComparator.INSTANCE)
           .forEach(lm -> {
-            dao.insertOrUpdate(dbSession, lm);
+            if (supportUpsert) {
+              dao.upsert(dbSession, lm);
+            } else {
+              dao.insertOrUpdate(dbSession, lm);
+            }
             metricIds.add(metric.getId());
             insertsOrUpdates++;
           });
       }
-      deleted += dao.deleteByComponentUuidExcludingMetricIds(dbSession, component.getUuid(), metricIds);
+      // The measures that no longer exist on the component must be deleted, for example
+      // when the coverage on a file goes to the "best value" 100%.
+      // The measures on deleted components are deleted by the step PurgeDatastoresStep
+      dao.deleteByComponentUuidExcludingMetricIds(dbSession, component.getUuid(), metricIds);
     }
   }
 

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java

@@ -111,7 +111,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
     assertThat(selectMeasure("module-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("module-value");
     assertThat(selectMeasure("dir-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("dir-value");
     assertThat(selectMeasure("file-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("file-value");
-    verifyStatistics(context, 4, 0);
+    verifyStatistics(context, 4);
   }
 
   @Test
@@ -125,7 +125,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
 
     assertThatMeasureIsNotPersisted("project-uuid", STRING_METRIC);
     assertThatMeasureIsNotPersisted("project-uuid", INT_METRIC);
-    verifyStatistics(context, 0, 0);
+    verifyStatistics(context, 0);
   }
 
   @Test
@@ -139,7 +139,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
     LiveMeasureDto persistedMeasure = selectMeasure("project-uuid", INT_METRIC).get();
     assertThat(persistedMeasure.getValue()).isNull();
     assertThat(persistedMeasure.getVariation()).isEqualTo(42.0);
-    verifyStatistics(context, 1, 0);
+    verifyStatistics(context, 1);
   }
 
   @Test
@@ -161,7 +161,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
     assertThatMeasureHasValue(measureOnFileInProject, 42);
     assertThatMeasureDoesNotExist(otherMeasureOnFileInProject);
     assertThatMeasureHasValue(measureInOtherProject, (int) measureInOtherProject.getValue().doubleValue());
-    verifyStatistics(context, 1, 1);
+    verifyStatistics(context, 1);
   }
 
   @Test
@@ -181,7 +181,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
 
     assertThatMeasureDoesNotExist(oldMeasure);
     assertThatMeasureHasValue("project-uuid", METRIC_WITH_BEST_VALUE, 0);
-    verifyStatistics(context, 1, 1);
+    verifyStatistics(context, 1);
   }
 
   @Test
@@ -200,7 +200,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
     assertThat(selectMeasure("view-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("view-value");
     assertThat(selectMeasure("subview-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("subview-value");
     assertThat(selectMeasure("project-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("project-value");
-    verifyStatistics(context, 3, 0);
+    verifyStatistics(context, 3);
   }
 
   private LiveMeasureDto insertMeasure(String componentUuid, String projectUuid, Metric metric) {
@@ -299,8 +299,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
     return new PersistLiveMeasuresStep(dbClient, metricRepository, new MeasureToMeasureDto(analysisMetadataHolder, treeRootHolder), treeRootHolder, measureRepository);
   }
 
-  private static void verifyStatistics(TestComputationStepContext context, int expectedInsertsOrUpdates, int expectedDeletes) {
+  private static void verifyStatistics(TestComputationStepContext context, int expectedInsertsOrUpdates) {
     context.getStatistics().assertValue("insertsOrUpdates", expectedInsertsOrUpdates);
-    context.getStatistics().assertValue("deletes", expectedDeletes);
   }
 }

server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java

@@ -31,6 +31,7 @@ import org.sonar.db.DbSession;
 import org.sonar.db.component.BranchType;
 import org.sonar.db.component.ComponentDto;
 import org.sonar.db.component.KeyType;
+import org.sonar.db.dialect.Dialect;
 
 import static java.util.Collections.singletonList;
 import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
@@ -104,6 +105,19 @@ public class LiveMeasureDao implements Dao {
     }
   }
 
+  /**
+   * Similar to {@link #insertOrUpdate(DbSession, LiveMeasureDto)}, except that:
+   * <ul>
+   * <li>it is batch session friendly (single same statement for both updates and inserts)</li>
+   * <li>it triggers a single SQL request</li>
+   * </ul>
+   * <p>
+   * <strong>This method should not be called unless {@link Dialect#supportsUpsert()} is true</strong>
+   */
+  public int upsert(DbSession dbSession, LiveMeasureDto dto) {
+    return mapper(dbSession).upsert(dto, Uuids.create(), system2.now());
+  }
+
   public int deleteByComponentUuidExcludingMetricIds(DbSession dbSession, String componentUuid, List<Integer> excludedMetricIds) {
     return mapper(dbSession).deleteByComponentUuidExcludingMetricIds(componentUuid, excludedMetricIds);
   }

server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java

@@ -60,6 +60,11 @@ public interface LiveMeasureMapper {
     @Param("dto") LiveMeasureDto dto,
     @Param("now") long now);
 
+  int upsert(
+    @Param("dto") LiveMeasureDto dto,
+    @Param("uuid") String uuid,
+    @Param("now") long now);
+
   int deleteByComponentUuidExcludingMetricIds(
     @Param("componentUuid") String componentUuid,
     @Param("excludedMetricIds") List<Integer> excludedMetricIds);

server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml

@@ -99,6 +99,65 @@
     and metric_id = #{dto.metricId, jdbcType=INTEGER}
   </update>
 
+  <update id="upsert" parameterType="map" useGeneratedKeys="false" databaseId="postgresql">
+    insert into live_measures (
+      uuid,
+      component_uuid,
+      project_uuid,
+      metric_id,
+      value,
+      text_value,
+      variation,
+      measure_data,
+      created_at,
+      updated_at
+    ) values (
+      #{uuid, jdbcType=VARCHAR},
+      #{dto.componentUuid, jdbcType=VARCHAR},
+      #{dto.projectUuid, jdbcType=VARCHAR},
+      #{dto.metricId, jdbcType=INTEGER},
+      #{dto.value, jdbcType=DOUBLE},
+      #{dto.textValue, jdbcType=VARCHAR},
+      #{dto.variation, jdbcType=DOUBLE},
+      #{dto.data, jdbcType=BINARY},
+      #{now, jdbcType=BIGINT},
+      #{now, jdbcType=BIGINT}
+    ) on conflict(component_uuid, metric_id) do update set
+      value = excluded.value,
+      variation = excluded.variation,
+      text_value = excluded.text_value,
+      measure_data  = excluded.measure_data,
+      updated_at = excluded.updated_at
+      where
+        <if test="dto.value==null">
+         live_measures.value is not null
+        </if>
+        <if test="dto.value!=null">
+          (live_measures.value is null or live_measures.value != excluded.value)
+        </if>
+        or
+        <if test="dto.textValue==null">
+          live_measures.text_value is not null
+        </if>
+        <if test="dto.textValue!=null">
+          (live_measures.text_value is null or live_measures.text_value != excluded.text_value)
+        </if>
+        or
+        <if test="dto.variation==null">
+          live_measures.variation is not null
+        </if>
+        <if test="dto.variation!=null">
+          (live_measures.variation is null or live_measures.variation != excluded.variation)
+        </if>
+        or
+        <if test="dto.data==null">
+          live_measures.measure_data is not null
+        </if>
+        <if test="dto.data!=null">
+          (live_measures.measure_data is null or live_measures.measure_data != excluded.measure_data)
+        </if>
+  </update>
+
   <delete id="deleteByComponentUuidExcludingMetricIds" parameterType="map">
     <include refid="sql_deleteByComponentUuidExcludingMetricIds"/>
   </delete>

server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java

@@ -22,7 +22,9 @@ package org.sonar.db.measure;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import org.apache.commons.lang.RandomStringUtils;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -38,6 +40,7 @@ import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assumptions.assumeThat;
 import static org.assertj.core.groups.Tuple.tuple;
 import static org.sonar.api.measures.Metric.ValueType.INT;
 import static org.sonar.db.component.ComponentTesting.newFileDto;
@@ -323,6 +326,184 @@ public class LiveMeasureDaoTest {
     assertThat(count).isEqualTo(1);
   }
 
+  @Test
+  public void deleteByComponentUuidExcludingMetricIds_with_empty_metrics() {
+    LiveMeasureDto measure1 = newLiveMeasure().setComponentUuid("C1").setMetricId(1);
+    LiveMeasureDto measure2 = newLiveMeasure().setComponentUuid("C1").setMetricId(2);
+    LiveMeasureDto measureOnOtherComponent = newLiveMeasure().setComponentUuid("C2").setMetricId(2);
+    underTest.insertOrUpdate(db.getSession(), measure1);
+    underTest.insertOrUpdate(db.getSession(), measure2);
+    underTest.insertOrUpdate(db.getSession(), measureOnOtherComponent);
+
+    int count = underTest.deleteByComponentUuidExcludingMetricIds(db.getSession(), "C1", Collections.emptyList());
+
+    assertThat(count).isEqualTo(2);
+    verifyTableSize(1);
+    verifyPersisted(measureOnOtherComponent);
+  }
+
+  @Test
+  public void upsert_inserts_or_updates_row() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+
+    // insert
+    LiveMeasureDto dto = newLiveMeasure();
+    int count = underTest.upsert(db.getSession(), dto);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+    assertThat(count).isEqualTo(1);
+
+    // update
+    dto.setValue(dto.getValue() + 1);
+    dto.setVariation(dto.getVariation() + 10);
+    dto.setData(dto.getDataAsString() + "_new");
+    count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(1);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_does_not_update_row_if_values_are_not_changed() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+
+    LiveMeasureDto dto = newLiveMeasure();
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(0);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_updates_row_if_lob_data_is_changed() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+
+    LiveMeasureDto dto = newLiveMeasure().setData(RandomStringUtils.random(10_000));
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    dto.setData(RandomStringUtils.random(dto.getDataAsString().length() + 10));
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(1);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_does_not_update_row_if_lob_data_is_not_changed() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+    LiveMeasureDto dto = newLiveMeasure().setData(RandomStringUtils.random(10_000));
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(0);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_updates_row_if_lob_data_is_removed() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+
+    LiveMeasureDto dto = newLiveMeasure().setData(RandomStringUtils.random(10_000));
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    dto.setData((String)null);
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(1);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_updates_row_if_variation_is_changed() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+    LiveMeasureDto dto = newLiveMeasure().setVariation(40.0);
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    dto.setVariation(50.0);
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(1);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_updates_row_if_variation_is_removed() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+    LiveMeasureDto dto = newLiveMeasure().setVariation(40.0);
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    dto.setVariation(null);
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(1);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_updates_row_if_variation_is_added() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+    LiveMeasureDto dto = newLiveMeasure().setVariation(null);
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    dto.setVariation(40.0);
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(1);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_updates_row_if_value_is_changed() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+    LiveMeasureDto dto = newLiveMeasure().setValue(40.0);
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    dto.setValue(50.0);
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(1);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_updates_row_if_value_is_removed() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+    LiveMeasureDto dto = newLiveMeasure().setValue(40.0);
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    dto.setValue(null);
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(1);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
+  @Test
+  public void upsert_updates_row_if_value_is_added() {
+    assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue();
+    LiveMeasureDto dto = newLiveMeasure().setValue(null);
+    underTest.upsert(db.getSession(), dto);
+
+    // update
+    dto.setValue(40.0);
+    int count = underTest.upsert(db.getSession(), dto);
+    assertThat(count).isEqualTo(1);
+    verifyPersisted(dto);
+    verifyTableSize(1);
+  }
+
   private void verifyTableSize(int expectedSize) {
     assertThat(db.countRowsOfTable(db.getSession(), "live_measures")).isEqualTo(expectedSize);
   }