SONAR-11814 speed-up upsert of live measures on PostgreSQL

diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java
index 44917e781444960da0ebc6436185381c65143d0a..a20eaeba14702c6d6ddf271b8100815a8c107c89 100644 (file)
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java
@@ -43,7 +43,7 @@ import org.sonar.ce.task.step.ComputationStep;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.measure.LiveMeasureComparator;
-import org.sonar.db.measure.LiveMeasureDao;
+import org.sonar.db.measure.LiveMeasureDto;
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.unmodifiableSet;
@@ -83,12 +83,13 @@ public class PersistLiveMeasuresStep implements ComputationStep {
   @Override
   public void execute(ComputationStep.Context context) {
     boolean supportUpsert = dbClient.getDatabase().getDialect().supportsUpsert();
-    try (DbSession dbSession = dbClient.openSession(supportUpsert)) {
+    try (DbSession dbSession = dbClient.openSession(false)) {
       Component root = treeRootHolder.getRoot();
       MeasureVisitor visitor = new MeasureVisitor(dbSession, supportUpsert);
       new DepthTraversalTypeAwareCrawler(visitor).visit(root);
 
-      context.getStatistics().add("insertsOrUpdates", visitor.insertsOrUpdates);
+      context.getStatistics()
+        .add("insertsOrUpdates", visitor.insertsOrUpdates);
     }
   }
 
@@ -105,9 +106,9 @@ public class PersistLiveMeasuresStep implements ComputationStep {
 
     @Override
     public void visitAny(Component component) {
-      LiveMeasureDao dao = dbClient.liveMeasureDao();
       List<Integer> metricIds = new ArrayList<>();
       Multimap<String, Measure> measures = measureRepository.getRawMeasures(component);
+      List<LiveMeasureDto> dtos = new ArrayList<>();
       for (Map.Entry<String, Collection<Measure>> measuresByMetricKey : measures.asMap().entrySet()) {
         String metricKey = measuresByMetricKey.getKey();
         if (NOT_TO_PERSIST_ON_FILE_METRIC_KEYS.contains(metricKey) && component.getType() == Component.Type.FILE) {
@@ -122,19 +123,23 @@ public class PersistLiveMeasuresStep implements ComputationStep {
           // To prevent deadlock, live measures are ordered the same way as in LiveMeasureComputerImpl#refreshComponentsOnSameProject
           .sorted(LiveMeasureComparator.INSTANCE)
           .forEach(lm -> {
-            if (supportUpsert) {
-              dao.upsert(dbSession, lm);
-            } else {
-              dao.insertOrUpdate(dbSession, lm);
-            }
+            dtos.add(lm);
             metricIds.add(metric.getId());
-            insertsOrUpdates++;
           });
       }
+      if (supportUpsert) {
+        dbClient.liveMeasureDao().upsert(dbSession, dtos);
+      } else {
+        for (LiveMeasureDto dto : dtos) {
+          dbClient.liveMeasureDao().insertOrUpdate(dbSession, dto);
+        }
+      }
+      insertsOrUpdates += dtos.size();
+
       // The measures that no longer exist on the component must be deleted, for example
       // when the coverage on a file goes to the "best value" 100%.
       // The measures on deleted components are deleted by the step PurgeDatastoresStep
-      dao.deleteByComponentUuidExcludingMetricIds(dbSession, component.getUuid(), metricIds);
+      dbClient.liveMeasureDao().deleteByComponentUuidExcludingMetricIds(dbSession, component.getUuid(), metricIds);
       dbSession.commit();
     }
   }
@@ -147,6 +152,4 @@ public class PersistLiveMeasuresStep implements ComputationStep {
       return input.getValueType() != Measure.ValueType.NO_VALUE || input.hasVariation() || input.getData() != null;
     }
   }
-
-
 }

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java
index 7f594fa7a37ac2e3fb6660a4b69bf016753589ff..2355e964c9152e03f424c58bdbeb14e0589190e7 100644 (file)
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.db.measure;
 
+import com.google.common.collect.Iterables;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -119,8 +120,15 @@ public class LiveMeasureDao implements Dao {
    * <p>
    * <strong>This method should not be called unless {@link Dialect#supportsUpsert()} is true</strong>
    */
-  public int upsert(DbSession dbSession, LiveMeasureDto dto) {
-    return mapper(dbSession).upsert(dto, Uuids.create(), system2.now());
+  public int upsert(DbSession dbSession, Iterable<LiveMeasureDto> dtos) {
+    for (LiveMeasureDto dto : dtos) {
+      dto.setUuidForUpsert(Uuids.create());
+    }
+    int updated = 0;
+    for (List<LiveMeasureDto> chunk : Iterables.partition(dtos, 100)) {
+      updated += mapper(dbSession).upsert(chunk, system2.now());
+    }
+    return updated;
   }
 
   public int deleteByComponentUuidExcludingMetricIds(DbSession dbSession, String componentUuid, List<Integer> excludedMetricIds) {

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDto.java
index 08984a4ad19eabafa42ea2214c359a7886dff623..a191cbf1d59dddbcebcb64b14b6848b1f08a2092 100644 (file)
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDto.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDto.java
@@ -28,6 +28,13 @@ public class LiveMeasureDto {
 
   private static final int MAX_TEXT_VALUE_LENGTH = 4000;
 
+  /**
+   * UUID generated only for UPSERT statements in PostgreSQL. It's never used
+   * in SELECT or regular INSERT/UPDATE.
+   */
+  @Nullable
+  private String uuidForUpsert;
+
   private String componentUuid;
   private String projectUuid;
   private int metricId;
@@ -40,6 +47,10 @@ public class LiveMeasureDto {
   @Nullable
   private Double variation;
 
+  void setUuidForUpsert(@Nullable String s) {
+    this.uuidForUpsert = s;
+  }
+
   public String getComponentUuid() {
     return componentUuid;
   }

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java
index caa8b9599e65c59da90ae35f065785a76fe44a29..3018e66afa3b671f8fe029b90a3ba0c7ba621626 100644 (file)
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java
@@ -63,10 +63,9 @@ public interface LiveMeasureMapper {
   int update(
     @Param("dto") LiveMeasureDto dto,
     @Param("now") long now);
-
+  
   int upsert(
-    @Param("dto") LiveMeasureDto dto,
-    @Param("uuid") String uuid,
+    @Param("dtos") List<LiveMeasureDto> dtos,
     @Param("now") long now);
 
   int deleteByComponentUuidExcludingMetricIds(

diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml
index a638e849c5265ac7c3075ef99fb7b6a8a5eb423d..41488372e3e532349215ab3aeb8371ae2afee73d 100644 (file)
--- a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml
+++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml
@@ -119,8 +119,9 @@
       measure_data,
       created_at,
       updated_at
-    ) values (
-      #{uuid, jdbcType=VARCHAR},
+    ) values
+    <foreach item="dto" collection="dtos" open="(" separator="),(" close=")">
+      #{dto.uuidForUpsert, jdbcType=VARCHAR},
       #{dto.componentUuid, jdbcType=VARCHAR},
       #{dto.projectUuid, jdbcType=VARCHAR},
       #{dto.metricId, jdbcType=INTEGER},
@@ -130,40 +131,18 @@
       #{dto.data, jdbcType=BINARY},
       #{now, jdbcType=BIGINT},
       #{now, jdbcType=BIGINT}
-    ) on conflict(component_uuid, metric_id) do update set
+    </foreach>
+    on conflict(component_uuid, metric_id) do update set
       value = excluded.value,
       variation = excluded.variation,
       text_value = excluded.text_value,
       measure_data  = excluded.measure_data,
       updated_at = excluded.updated_at
-      where
-        <if test="dto.value==null">
-         live_measures.value is not null
-        </if>
-        <if test="dto.value!=null">
-          (live_measures.value is null or live_measures.value != excluded.value)
-        </if>
-        or
-        <if test="dto.textValue==null">
-          live_measures.text_value is not null
-        </if>
-        <if test="dto.textValue!=null">
-          (live_measures.text_value is null or live_measures.text_value != excluded.text_value)
-        </if>
-        or
-        <if test="dto.variation==null">
-          live_measures.variation is not null
-        </if>
-        <if test="dto.variation!=null">
-          (live_measures.variation is null or live_measures.variation != excluded.variation)
-        </if>
-        or
-        <if test="dto.data==null">
-          live_measures.measure_data is not null
-        </if>
-        <if test="dto.data!=null">
-          (live_measures.measure_data is null or live_measures.measure_data != excluded.measure_data)
-        </if>
+    where
+      live_measures.value is distinct from excluded.value or
+      live_measures.variation is distinct from excluded.variation or
+      live_measures.text_value is distinct from excluded.text_value or
+      live_measures.measure_data is distinct from excluded.measure_data
   </update>
 
   <delete id="deleteByComponentUuidExcludingMetricIds" parameterType="map">

diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java
index 0c6561d986f51d3cb46583f42956517d14d87cb0..f55cc9b7a24f9fbd9ac4e767775331da96a616e9 100644 (file)
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.IntStream;
 import org.apache.commons.lang.RandomStringUtils;
 import org.junit.Before;
 import org.junit.Rule;
@@ -377,7 +378,7 @@ public class LiveMeasureDaoTest {
 
     // insert
     LiveMeasureDto dto = newLiveMeasure();
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     verifyPersisted(dto);
     verifyTableSize(1);
     assertThat(count).isEqualTo(1);
@@ -386,7 +387,7 @@ public class LiveMeasureDaoTest {
     dto.setValue(dto.getValue() + 1);
     dto.setVariation(dto.getVariation() + 10);
     dto.setData(dto.getDataAsString() + "_new");
-    count = underTest.upsert(db.getSession(), dto);
+    count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(1);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -399,10 +400,10 @@ public class LiveMeasureDaoTest {
     }
 
     LiveMeasureDto dto = newLiveMeasure();
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(0);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -415,11 +416,11 @@ public class LiveMeasureDaoTest {
     }
 
     LiveMeasureDto dto = newLiveMeasure().setData(RandomStringUtils.random(10_000));
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
     dto.setData(RandomStringUtils.random(dto.getDataAsString().length() + 10));
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(1);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -431,10 +432,10 @@ public class LiveMeasureDaoTest {
       return;
     }
     LiveMeasureDto dto = newLiveMeasure().setData(RandomStringUtils.random(10_000));
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(0);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -447,11 +448,11 @@ public class LiveMeasureDaoTest {
     }
 
     LiveMeasureDto dto = newLiveMeasure().setData(RandomStringUtils.random(10_000));
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
     dto.setData((String)null);
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(1);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -463,11 +464,11 @@ public class LiveMeasureDaoTest {
       return;
     }
     LiveMeasureDto dto = newLiveMeasure().setVariation(40.0);
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
     dto.setVariation(50.0);
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(1);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -479,11 +480,11 @@ public class LiveMeasureDaoTest {
       return;
     }
     LiveMeasureDto dto = newLiveMeasure().setVariation(40.0);
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
     dto.setVariation(null);
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(1);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -495,11 +496,11 @@ public class LiveMeasureDaoTest {
       return;
     }
     LiveMeasureDto dto = newLiveMeasure().setVariation(null);
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
     dto.setVariation(40.0);
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(1);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -511,11 +512,11 @@ public class LiveMeasureDaoTest {
       return;
     }
     LiveMeasureDto dto = newLiveMeasure().setValue(40.0);
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
     dto.setValue(50.0);
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(1);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -527,11 +528,11 @@ public class LiveMeasureDaoTest {
       return;
     }
     LiveMeasureDto dto = newLiveMeasure().setValue(40.0);
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
     dto.setValue(null);
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(1);
     verifyPersisted(dto);
     verifyTableSize(1);
@@ -543,16 +544,42 @@ public class LiveMeasureDaoTest {
       return;
     }
     LiveMeasureDto dto = newLiveMeasure().setValue(null);
-    underTest.upsert(db.getSession(), dto);
+    underTest.upsert(db.getSession(), asList(dto));
 
     // update
     dto.setValue(40.0);
-    int count = underTest.upsert(db.getSession(), dto);
+    int count = underTest.upsert(db.getSession(), asList(dto));
     assertThat(count).isEqualTo(1);
     verifyPersisted(dto);
     verifyTableSize(1);
   }
 
+  @Test
+  public void upsert_multiple_rows() {
+    if (!db.getDbClient().getDatabase().getDialect().supportsUpsert()) {
+      return;
+    }
+
+    // insert 30
+    List<LiveMeasureDto> inserted = new ArrayList<>();
+    IntStream.range(0, 30).forEach(i -> inserted.add(newLiveMeasure()));
+    int result = underTest.upsert(db.getSession(), inserted);
+    verifyTableSize(30);
+    assertThat(result).isEqualTo(30);
+
+    // update 10 with new values, update 5 without any change and insert new 50
+    List<LiveMeasureDto> upserted = new ArrayList<>();
+    IntStream.range(0, 10).forEach(i -> {
+      LiveMeasureDto d = inserted.get(i);
+      upserted.add(d.setValue(d.getValue() + 123));
+    });
+    upserted.addAll(inserted.subList(10, 15));
+    IntStream.range(0, 50).forEach(i -> upserted.add(newLiveMeasure()));
+    result = underTest.upsert(db.getSession(), upserted);
+    verifyTableSize(80);
+    assertThat(result).isEqualTo(60);
+  }
+
   private void verifyTableSize(int expectedSize) {
     assertThat(db.countRowsOfTable(db.getSession(), "live_measures")).isEqualTo(expectedSize);
   }
@@ -560,6 +587,8 @@ public class LiveMeasureDaoTest {
   private void verifyPersisted(LiveMeasureDto dto) {
     List<LiveMeasureDto> selected = underTest.selectByComponentUuidsAndMetricIds(db.getSession(), singletonList(dto.getComponentUuid()), singletonList(dto.getMetricId()));
     assertThat(selected).hasSize(1);
-    assertThat(selected.get(0)).isEqualToComparingFieldByField(dto);
+    assertThat(selected.get(0)).isEqualToComparingOnlyGivenFields(dto,
+      // do not compare the field "uuid", which is used only for insert, not select
+      "componentUuid", "projectUuid", "metricId", "value", "textValue", "data", "variation");
   }
 }