From 020d9116cf51f1e5bac804d0c8d797e28cfba1e9 Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Lesaint?= Date: Thu, 4 Jul 2019 10:14:24 +0200 Subject: [PATCH] SONAR-12251 detect aborted attempt during processing of 1st project --- .../db/migration/charset/ColumnDef.java | 2 +- .../version/v70/PopulateLiveMeasures.java | 34 +++++++++++++------ .../version/v70/PopulateLiveMeasuresTest.java | 10 +++++- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java index 386ed39ce09..134f3b5e90f 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/charset/ColumnDef.java @@ -90,7 +90,7 @@ public class ColumnDef { @Override public ColumnDef convert(ResultSet rs) throws SQLException { String nullableText = rs.getString(7); - boolean nullable = "YES".equalsIgnoreCase(nullableText); + boolean nullable = "FIRST".equalsIgnoreCase(nullableText); return new ColumnDef( rs.getString(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getString(5), rs.getLong(6), nullable); diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v70/PopulateLiveMeasures.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v70/PopulateLiveMeasures.java index f82d1ea6740..76405c389eb 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v70/PopulateLiveMeasures.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v70/PopulateLiveMeasures.java @@ -54,9 +54,9 @@ public class PopulateLiveMeasures extends DataChange { @Override protected void execute(Context context) throws SQLException { - boolean firstAttempt = isFirstAttempt(context); - if (!firstAttempt) { - LOG.info("Retry detected (non empty table live_measures_p). Handle it"); + ATTEMPT attempt = isFirstAttempt(context); + if (attempt != ATTEMPT.FIRST) { + LOG.info("Retry detected (non empty table live_measures_p or live_measures). Handling it"); } long now = system2.now(); @@ -67,32 +67,44 @@ public class PopulateLiveMeasures extends DataChange { " from snapshots s" + " where" + " s.islast = ?" + - (firstAttempt ? "" : " and not exists (select 1 from live_measures_p lmp where lmp.project_uuid=s.component_uuid)"); + (attempt == ATTEMPT.ONE_PROJECT_OR_MORE ? " and not exists (select 1 from live_measures_p lmp where lmp.project_uuid=s.component_uuid)" : ""); try (Connection connection = createReadUncommittedConnection(); Select select = SelectImpl.create(db, connection, statement).setBoolean(1, true)) { List rows = new ArrayList<>(projectBatchSize); select.scroll(t -> { rows.add(new Row(t.getString(1), t.getString(2))); if (rows.size() == projectBatchSize) { - processProjectBatch(context, rows, firstAttempt, now); + processProjectBatch(context, rows, attempt, now); rows.clear(); } }); if (!rows.isEmpty()) { - processProjectBatch(context, rows, firstAttempt, now); + processProjectBatch(context, rows, attempt, now); } } } - private static boolean isFirstAttempt(Context context) throws SQLException { - try (Select select = context.prepareSelect("select count(1) from live_measures_p")) { - return select.get(t -> t.getLong(1)) == 0; + private static ATTEMPT isFirstAttempt(Context context) throws SQLException { + try ( + Select selectFromTempTable = context.prepareSelect("select count(1) from live_measures_p"); + Select selectFromLiveMeasures = context.prepareSelect("select count(1) from live_measures") + ) { + boolean projectsProcessed = selectFromTempTable.get(t -> t.getLong(1)) > 0; + if (projectsProcessed) { + return ATTEMPT.ONE_PROJECT_OR_MORE; + } + // we will count at most the data of one project => table should have little content => bad performance risk is low + return selectFromLiveMeasures.get(t -> t.getLong(1)) == 0 ? ATTEMPT.FIRST : ATTEMPT.PARTIAL; } } - private static void processProjectBatch(Context context, List rows, boolean firstAttempt, long now) throws SQLException { + private enum ATTEMPT { + FIRST, PARTIAL, ONE_PROJECT_OR_MORE + } + + private static void processProjectBatch(Context context, List rows, ATTEMPT attempt, long now) throws SQLException { MassUpdate massUpdate = context.prepareMassUpdate(); massUpdate.rowPluralName("live measures"); setSelect(rows, massUpdate); @@ -104,7 +116,7 @@ public class PopulateLiveMeasures extends DataChange { massUpdate.update("insert into live_measures_p (project_uuid) values (?)") // we want to commit each project finished asap to avoid restarting them over in case of interruption .setBatchSize(1); - LiveMeasurePopulationMultiHandler handler = new LiveMeasurePopulationMultiHandler(firstAttempt, rows, now); + LiveMeasurePopulationMultiHandler handler = new LiveMeasurePopulationMultiHandler(attempt == ATTEMPT.FIRST, rows, now); massUpdate.execute(handler); Set notCommittedProjectUuids = handler.notCommittedProjectUuids; diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v70/PopulateLiveMeasuresTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v70/PopulateLiveMeasuresTest.java index 6368c94bfaf..e25e9e8410c 100644 --- a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v70/PopulateLiveMeasuresTest.java +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v70/PopulateLiveMeasuresTest.java @@ -87,7 +87,7 @@ public class PopulateLiveMeasuresTest { } @Test - public void do_not_fail_if_live_measure_of_component_already_partially_inserted() throws SQLException { + public void migration_is_reentrant_on_partially_processed_1st_project() throws SQLException { generateProjectMeasures(); db.executeInsert( @@ -102,6 +102,14 @@ public class PopulateLiveMeasuresTest { underTest.execute(); + assertThat(getLiveMeasures()).extracting( + field("COMPONENT_UUID"), + field("PROJECT_UUID"), + field("METRIC_ID"), + field("VALUE"), + field("TEXT_VALUE"), + field("VARIATION"), + field("MEASURE_DATA")).containsExactlyInAnyOrder(generateLiveMeasures()); } private Function, Object> field(String name) { -- 2.39.5