From 3df6519965c9409e5e13de6bc0ed6a703863fd85 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Mon, 30 Jul 2018 15:06:39 +0200 Subject: [PATCH] SONAR-11077 add statistics to CE PersistLiveMeasuresStep --- .../step/PersistLiveMeasuresStep.java | 10 +++++-- .../step/PersistLiveMeasuresStepTest.java | 28 +++++++++++++++---- .../org/sonar/db/measure/LiveMeasureDao.java | 4 +-- .../sonar/db/measure/LiveMeasureMapper.java | 2 +- .../sonar/db/measure/LiveMeasureDaoTest.java | 3 +- 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java index 536cd39183f..4b50f7343fd 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java @@ -87,15 +87,20 @@ public class PersistLiveMeasuresStep implements ComputationStep { try (DbSession dbSession = dbClient.openSession(false)) { String marker = Uuids.create(); Component root = treeRootHolder.getRoot(); - new DepthTraversalTypeAwareCrawler(new MeasureVisitor(dbSession, marker)).visit(root); - dbClient.liveMeasureDao().deleteByProjectUuidExcludingMarker(dbSession, root.getUuid(), marker); + MeasureVisitor visitor = new MeasureVisitor(dbSession, marker); + new DepthTraversalTypeAwareCrawler(visitor).visit(root); + int deleted = dbClient.liveMeasureDao().deleteByProjectUuidExcludingMarker(dbSession, root.getUuid(), marker); dbSession.commit(); + + context.getStatistics().add("insertsOrUpdates", visitor.total); + context.getStatistics().add("deletes", deleted); } } private class MeasureVisitor extends TypeAwareVisitorAdapter { private final DbSession dbSession; private final String marker; + private int total = 0; private MeasureVisitor(DbSession dbSession, String marker) { super(CrawlerDepthLimit.LEAVES, PRE_ORDER); @@ -123,6 +128,7 @@ public class PersistLiveMeasuresStep implements ComputationStep { while (liveMeasures.hasNext()) { dao.insertOrUpdate(dbSession, liveMeasures.next(), marker); count++; + total++; if (count % 100 == 0) { // use short transactions to avoid potential deadlocks on MySQL // https://jira.sonarsource.com/browse/SONAR-10117?focusedCommentId=153555&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-153555 diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java index 6d709263662..a9e6aaf9205 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java @@ -101,7 +101,8 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("dir-value")); measureRepository.addRawMeasure(REF_4, STRING_METRIC.getKey(), newMeasureBuilder().create("file-value")); - step().execute(new TestComputationStepContext()); + TestComputationStepContext context = new TestComputationStepContext(); + step().execute(context); // all measures are persisted, from project to file assertThat(db.countRowsOfTable("live_measures")).isEqualTo(4); @@ -109,6 +110,7 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { assertThat(selectMeasure("module-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("module-value"); assertThat(selectMeasure("dir-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("dir-value"); assertThat(selectMeasure("file-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("file-value"); + verifyStatistics(context, 4, 0); } @Test @@ -117,10 +119,12 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { measureRepository.addRawMeasure(REF_1, STRING_METRIC.getKey(), newMeasureBuilder().createNoValue()); measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().createNoValue()); - step().execute(new TestComputationStepContext()); + TestComputationStepContext context = new TestComputationStepContext(); + step().execute(context); assertThatMeasureIsNotPersisted("project-uuid", STRING_METRIC); assertThatMeasureIsNotPersisted("project-uuid", INT_METRIC); + verifyStatistics(context, 0, 0); } @Test @@ -128,11 +132,13 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { prepareProject(); measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().setVariation(42.0).createNoValue()); - step().execute(new TestComputationStepContext()); + TestComputationStepContext context = new TestComputationStepContext(); + step().execute(context); LiveMeasureDto persistedMeasure = selectMeasure("project-uuid", INT_METRIC).get(); assertThat(persistedMeasure.getValue()).isNull(); assertThat(persistedMeasure.getVariation()).isEqualTo(42.0); + verifyStatistics(context, 1, 0); } @Test @@ -150,12 +156,14 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { measureRepository.addRawMeasure(REF_4, INT_METRIC.getKey(), newMeasureBuilder().create(42)); - step().execute(new TestComputationStepContext()); + TestComputationStepContext context = new TestComputationStepContext(); + step().execute(context); assertThatMeasureHasValue(measureOnFileInProject, 42); assertThatMeasureDoesNotExist(measureOnDeletedFileInProject); assertThatMeasureDoesNotExist(otherMeasureOnFileInProject); assertThatMeasureHasValue(measureInOtherProject, (int) measureInOtherProject.getValue().doubleValue()); + verifyStatistics(context, 1, 2); } @Test @@ -170,10 +178,12 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { // file measure with metric best value -> do not persist measureRepository.addRawMeasure(REF_4, METRIC_WITH_BEST_VALUE.getKey(), newMeasureBuilder().create(0)); - step().execute(new TestComputationStepContext()); + TestComputationStepContext context = new TestComputationStepContext(); + step().execute(context); assertThatMeasureDoesNotExist(oldMeasure); assertThatMeasureHasValue("project-uuid", METRIC_WITH_BEST_VALUE, 0); + verifyStatistics(context, 1, 1); } @Test @@ -185,12 +195,14 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { measureRepository.addRawMeasure(REF_2, STRING_METRIC.getKey(), newMeasureBuilder().create("subview-value")); measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("project-value")); - step().execute(new TestComputationStepContext()); + TestComputationStepContext context = new TestComputationStepContext(); + step().execute(context); assertThat(db.countRowsOfTable("live_measures")).isEqualTo(3); assertThat(selectMeasure("view-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("view-value"); assertThat(selectMeasure("subview-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("subview-value"); assertThat(selectMeasure("project-uuid", STRING_METRIC).get().getDataAsString()).isEqualTo("project-value"); + verifyStatistics(context, 3, 0); } private LiveMeasureDto insertMeasure(String componentUuid, String projectUuid, Metric metric) { @@ -289,4 +301,8 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { return new PersistLiveMeasuresStep(dbClient, metricRepository, new MeasureToMeasureDto(analysisMetadataHolder, treeRootHolder), treeRootHolder, measureRepository); } + private static void verifyStatistics(TestComputationStepContext context, int expectedInsertsOrUpdates, int expectedDeletes) { + context.getStatistics().assertValue("insertsOrUpdates", expectedInsertsOrUpdates); + context.getStatistics().assertValue("deletes", expectedDeletes); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java index 4e403f3a3aa..3e511b04519 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java @@ -108,8 +108,8 @@ public class LiveMeasureDao implements Dao { /** * Delete the rows that do NOT have the specified marker */ - public void deleteByProjectUuidExcludingMarker(DbSession dbSession, String projectUuid, String marker) { - mapper(dbSession).deleteByProjectUuidExcludingMarker(projectUuid, marker); + public int deleteByProjectUuidExcludingMarker(DbSession dbSession, String projectUuid, String marker) { + return mapper(dbSession).deleteByProjectUuidExcludingMarker(projectUuid, marker); } private static LiveMeasureMapper mapper(DbSession dbSession) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java index afd60719696..73affd0fd1a 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java @@ -62,7 +62,7 @@ public interface LiveMeasureMapper { @Nullable @Param("marker") String marker, @Param("now") long now); - void deleteByProjectUuidExcludingMarker( + int deleteByProjectUuidExcludingMarker( @Param("projectUuid") String projectUuid, @Param("marker") String marker); } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java index 83fb50719ab..9abd6463d49 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java @@ -315,12 +315,13 @@ public class LiveMeasureDaoTest { underTest.insertOrUpdate(db.getSession(), measure4NoMarker, null); underTest.insertOrUpdate(db.getSession(), measure5OtherProject, "foo"); - underTest.deleteByProjectUuidExcludingMarker(db.getSession(), "P1", "foo"); + int count = underTest.deleteByProjectUuidExcludingMarker(db.getSession(), "P1", "foo"); verifyTableSize(3); verifyPersisted(measure1); verifyPersisted(measure2); verifyPersisted(measure5OtherProject); + assertThat(count).isEqualTo(2); } private void verifyTableSize(int expectedSize) { -- 2.39.5