From 45583f8ad0ba7c1167704214bb790c8c1fd755be Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Mon, 29 Oct 2018 08:38:25 +0100 Subject: [PATCH] SONARCLOUD-161 upsert live measures on postgresql --- .../step/PersistLiveMeasuresStep.java | 22 ++- .../step/PersistLiveMeasuresStepTest.java | 15 +- .../org/sonar/db/measure/LiveMeasureDao.java | 14 ++ .../sonar/db/measure/LiveMeasureMapper.java | 5 + .../sonar/db/measure/LiveMeasureMapper.xml | 59 ++++++ .../sonar/db/measure/LiveMeasureDaoTest.java | 181 ++++++++++++++++++ 6 files changed, 281 insertions(+), 15 deletions(-) diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java index 4976f7a5c25..497b5dd2e33 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java @@ -84,24 +84,25 @@ public class PersistLiveMeasuresStep implements ComputationStep { @Override public void execute(ComputationStep.Context context) { - try (DbSession dbSession = dbClient.openSession(false)) { + boolean supportUpsert = dbClient.getDatabase().getDialect().supportsUpsert(); + try (DbSession dbSession = dbClient.openSession(supportUpsert)) { Component root = treeRootHolder.getRoot(); - MeasureVisitor visitor = new MeasureVisitor(dbSession); + MeasureVisitor visitor = new MeasureVisitor(dbSession, supportUpsert); new DepthTraversalTypeAwareCrawler(visitor).visit(root); dbSession.commit(); context.getStatistics().add("insertsOrUpdates", visitor.insertsOrUpdates); - context.getStatistics().add("deletes", visitor.deleted); } } private class MeasureVisitor extends TypeAwareVisitorAdapter { private final DbSession dbSession; + private final boolean supportUpsert; private int insertsOrUpdates = 0; - private int deleted = 0; - private MeasureVisitor(DbSession dbSession) { + private MeasureVisitor(DbSession dbSession, boolean supportUpsert) { super(CrawlerDepthLimit.LEAVES, PRE_ORDER); + this.supportUpsert = supportUpsert; this.dbSession = dbSession; } @@ -124,12 +125,19 @@ public class PersistLiveMeasuresStep implements ComputationStep { // To prevent deadlock, live measures are ordered the same way as in LiveMeasureComputerImpl#refreshComponentsOnSameProject .sorted(LiveMeasureComparator.INSTANCE) .forEach(lm -> { - dao.insertOrUpdate(dbSession, lm); + if (supportUpsert) { + dao.upsert(dbSession, lm); + } else { + dao.insertOrUpdate(dbSession, lm); + } metricIds.add(metric.getId()); insertsOrUpdates++; }); } - deleted += dao.deleteByComponentUuidExcludingMetricIds(dbSession, component.getUuid(), metricIds); + // The measures that no longer exist on the component must be deleted, for example + // when the coverage on a file goes to the "best value" 100%. + // The measures on deleted components are deleted by the step PurgeDatastoresStep + dao.deleteByComponentUuidExcludingMetricIds(dbSession, component.getUuid(), metricIds); } } diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java index fdf6a824f20..2b89a90f9e6 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java @@ -111,7 +111,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { assertThat(selectMeasure("module-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("module-value"); assertThat(selectMeasure("dir-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("dir-value"); assertThat(selectMeasure("file-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("file-value"); - verifyStatistics(context, 4, 0); + verifyStatistics(context, 4); } @Test @@ -125,7 +125,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { assertThatMeasureIsNotPersisted("project-uuid", STRING_METRIC); assertThatMeasureIsNotPersisted("project-uuid", INT_METRIC); - verifyStatistics(context, 0, 0); + verifyStatistics(context, 0); } @Test @@ -139,7 +139,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { LiveMeasureDto persistedMeasure = selectMeasure("project-uuid", INT_METRIC).get(); assertThat(persistedMeasure.getValue()).isNull(); assertThat(persistedMeasure.getVariation()).isEqualTo(42.0); - verifyStatistics(context, 1, 0); + verifyStatistics(context, 1); } @Test @@ -161,7 +161,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { assertThatMeasureHasValue(measureOnFileInProject, 42); assertThatMeasureDoesNotExist(otherMeasureOnFileInProject); assertThatMeasureHasValue(measureInOtherProject, (int) measureInOtherProject.getValue().doubleValue()); - verifyStatistics(context, 1, 1); + verifyStatistics(context, 1); } @Test @@ -181,7 +181,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { assertThatMeasureDoesNotExist(oldMeasure); assertThatMeasureHasValue("project-uuid", METRIC_WITH_BEST_VALUE, 0); - verifyStatistics(context, 1, 1); + verifyStatistics(context, 1); } @Test @@ -200,7 +200,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { assertThat(selectMeasure("view-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("view-value"); assertThat(selectMeasure("subview-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("subview-value"); assertThat(selectMeasure("project-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("project-value"); - verifyStatistics(context, 3, 0); + verifyStatistics(context, 3); } private LiveMeasureDto insertMeasure(String componentUuid, String projectUuid, Metric metric) { @@ -299,8 +299,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { return new PersistLiveMeasuresStep(dbClient, metricRepository, new MeasureToMeasureDto(analysisMetadataHolder, treeRootHolder), treeRootHolder, measureRepository); } - private static void verifyStatistics(TestComputationStepContext context, int expectedInsertsOrUpdates, int expectedDeletes) { + private static void verifyStatistics(TestComputationStepContext context, int expectedInsertsOrUpdates) { context.getStatistics().assertValue("insertsOrUpdates", expectedInsertsOrUpdates); - context.getStatistics().assertValue("deletes", expectedDeletes); } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java index 4a2e12c5ba1..9fc5e518ba5 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java @@ -31,6 +31,7 @@ import org.sonar.db.DbSession; import org.sonar.db.component.BranchType; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.KeyType; +import org.sonar.db.dialect.Dialect; import static java.util.Collections.singletonList; import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; @@ -104,6 +105,19 @@ public class LiveMeasureDao implements Dao { } } + /** + * Similar to {@link #insertOrUpdate(DbSession, LiveMeasureDto)}, except that: + * + *

+ * This method should not be called unless {@link Dialect#supportsUpsert()} is true + */ + public int upsert(DbSession dbSession, LiveMeasureDto dto) { + return mapper(dbSession).upsert(dto, Uuids.create(), system2.now()); + } + public int deleteByComponentUuidExcludingMetricIds(DbSession dbSession, String componentUuid, List excludedMetricIds) { return mapper(dbSession).deleteByComponentUuidExcludingMetricIds(componentUuid, excludedMetricIds); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java index d45d8068e84..8ffb21bec40 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java @@ -60,6 +60,11 @@ public interface LiveMeasureMapper { @Param("dto") LiveMeasureDto dto, @Param("now") long now); + int upsert( + @Param("dto") LiveMeasureDto dto, + @Param("uuid") String uuid, + @Param("now") long now); + int deleteByComponentUuidExcludingMetricIds( @Param("componentUuid") String componentUuid, @Param("excludedMetricIds") List excludedMetricIds); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml index 526d2280c7c..36c18681742 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml @@ -99,6 +99,65 @@ and metric_id = #{dto.metricId, jdbcType=INTEGER} + + insert into live_measures ( + uuid, + component_uuid, + project_uuid, + metric_id, + value, + text_value, + variation, + measure_data, + created_at, + updated_at + ) values ( + #{uuid, jdbcType=VARCHAR}, + #{dto.componentUuid, jdbcType=VARCHAR}, + #{dto.projectUuid, jdbcType=VARCHAR}, + #{dto.metricId, jdbcType=INTEGER}, + #{dto.value, jdbcType=DOUBLE}, + #{dto.textValue, jdbcType=VARCHAR}, + #{dto.variation, jdbcType=DOUBLE}, + #{dto.data, jdbcType=BINARY}, + #{now, jdbcType=BIGINT}, + #{now, jdbcType=BIGINT} + ) on conflict(component_uuid, metric_id) do update set + value = excluded.value, + variation = excluded.variation, + text_value = excluded.text_value, + measure_data = excluded.measure_data, + updated_at = excluded.updated_at + where + + live_measures.value is not null + + + (live_measures.value is null or live_measures.value != excluded.value) + + or + + live_measures.text_value is not null + + + (live_measures.text_value is null or live_measures.text_value != excluded.text_value) + + or + + live_measures.variation is not null + + + (live_measures.variation is null or live_measures.variation != excluded.variation) + + or + + live_measures.measure_data is not null + + + (live_measures.measure_data is null or live_measures.measure_data != excluded.measure_data) + + + diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java index 659379525d2..291ff72290f 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java @@ -22,7 +22,9 @@ package org.sonar.db.measure; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import org.apache.commons.lang.RandomStringUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -38,6 +40,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; import static org.assertj.core.groups.Tuple.tuple; import static org.sonar.api.measures.Metric.ValueType.INT; import static org.sonar.db.component.ComponentTesting.newFileDto; @@ -323,6 +326,184 @@ public class LiveMeasureDaoTest { assertThat(count).isEqualTo(1); } + @Test + public void deleteByComponentUuidExcludingMetricIds_with_empty_metrics() { + LiveMeasureDto measure1 = newLiveMeasure().setComponentUuid("C1").setMetricId(1); + LiveMeasureDto measure2 = newLiveMeasure().setComponentUuid("C1").setMetricId(2); + LiveMeasureDto measureOnOtherComponent = newLiveMeasure().setComponentUuid("C2").setMetricId(2); + underTest.insertOrUpdate(db.getSession(), measure1); + underTest.insertOrUpdate(db.getSession(), measure2); + underTest.insertOrUpdate(db.getSession(), measureOnOtherComponent); + + int count = underTest.deleteByComponentUuidExcludingMetricIds(db.getSession(), "C1", Collections.emptyList()); + + assertThat(count).isEqualTo(2); + verifyTableSize(1); + verifyPersisted(measureOnOtherComponent); + } + + @Test + public void upsert_inserts_or_updates_row() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + + // insert + LiveMeasureDto dto = newLiveMeasure(); + int count = underTest.upsert(db.getSession(), dto); + verifyPersisted(dto); + verifyTableSize(1); + assertThat(count).isEqualTo(1); + + // update + dto.setValue(dto.getValue() + 1); + dto.setVariation(dto.getVariation() + 10); + dto.setData(dto.getDataAsString() + "_new"); + count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_does_not_update_row_if_values_are_not_changed() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + + LiveMeasureDto dto = newLiveMeasure(); + underTest.upsert(db.getSession(), dto); + + // update + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(0); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_updates_row_if_lob_data_is_changed() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + + LiveMeasureDto dto = newLiveMeasure().setData(RandomStringUtils.random(10_000)); + underTest.upsert(db.getSession(), dto); + + // update + dto.setData(RandomStringUtils.random(dto.getDataAsString().length() + 10)); + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_does_not_update_row_if_lob_data_is_not_changed() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + LiveMeasureDto dto = newLiveMeasure().setData(RandomStringUtils.random(10_000)); + underTest.upsert(db.getSession(), dto); + + // update + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(0); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_updates_row_if_lob_data_is_removed() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + + LiveMeasureDto dto = newLiveMeasure().setData(RandomStringUtils.random(10_000)); + underTest.upsert(db.getSession(), dto); + + // update + dto.setData((String)null); + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_updates_row_if_variation_is_changed() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + LiveMeasureDto dto = newLiveMeasure().setVariation(40.0); + underTest.upsert(db.getSession(), dto); + + // update + dto.setVariation(50.0); + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_updates_row_if_variation_is_removed() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + LiveMeasureDto dto = newLiveMeasure().setVariation(40.0); + underTest.upsert(db.getSession(), dto); + + // update + dto.setVariation(null); + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_updates_row_if_variation_is_added() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + LiveMeasureDto dto = newLiveMeasure().setVariation(null); + underTest.upsert(db.getSession(), dto); + + // update + dto.setVariation(40.0); + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_updates_row_if_value_is_changed() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + LiveMeasureDto dto = newLiveMeasure().setValue(40.0); + underTest.upsert(db.getSession(), dto); + + // update + dto.setValue(50.0); + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_updates_row_if_value_is_removed() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + LiveMeasureDto dto = newLiveMeasure().setValue(40.0); + underTest.upsert(db.getSession(), dto); + + // update + dto.setValue(null); + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyPersisted(dto); + verifyTableSize(1); + } + + @Test + public void upsert_updates_row_if_value_is_added() { + assumeThat(db.getDbClient().getDatabase().getDialect().supportsUpsert()).isTrue(); + LiveMeasureDto dto = newLiveMeasure().setValue(null); + underTest.upsert(db.getSession(), dto); + + // update + dto.setValue(40.0); + int count = underTest.upsert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyPersisted(dto); + verifyTableSize(1); + } + private void verifyTableSize(int expectedSize) { assertThat(db.countRowsOfTable(db.getSession(), "live_measures")).isEqualTo(expectedSize); } -- 2.39.5