From 0633a01ff3c882cc438773f45981563c647335aa Mon Sep 17 00:00:00 2001 From: =?utf8?q?L=C3=A9o=20Geoffroy?= Date: Tue, 29 Oct 2024 08:32:06 +0100 Subject: [PATCH] SONAR-23998 Add migration of deprecated json metrics to measures migration fixup! SONAR-23998 Add migration of deprecated json metrics to measures migration SONAR-23998 Add a unit test to check on cenversion failure --- ...grateBranchesLiveMeasuresToMeasuresIT.java | 79 +++++++++++++++++++ ...AbstractMigrateLiveMeasuresToMeasures.java | 17 +++- .../version/v108/MeasureMigration.java | 62 +++++++++++++++ 3 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/MeasureMigration.java diff --git a/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/MigrateBranchesLiveMeasuresToMeasuresIT.java b/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/MigrateBranchesLiveMeasuresToMeasuresIT.java index a3ff63478f3..74813eea032 100644 --- a/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/MigrateBranchesLiveMeasuresToMeasuresIT.java +++ b/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/MigrateBranchesLiveMeasuresToMeasuresIT.java @@ -19,6 +19,7 @@ */ package org.sonar.server.platform.db.migration.version.v108; +import com.google.gson.Gson; import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.HashMap; @@ -29,8 +30,10 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.event.Level; +import org.sonar.api.measures.CoreMetrics; import org.sonar.api.testfixtures.log.LogTesterJUnit5; import org.sonar.api.utils.System2; +import org.sonar.core.metric.SoftwareQualitiesMetrics; import org.sonar.core.util.SequenceUuidFactory; import org.sonar.db.MigrationDbTester; import org.sonar.server.platform.db.migration.step.DataChange; @@ -190,6 +193,82 @@ class MigrateBranchesLiveMeasuresToMeasuresIT { .isEmpty(); } + @Test + void should_include_new_measures_based_on_previous_available_measures() throws SQLException { + Set metricsToMigrate = MeasureMigration.MIGRATION_MAP.keySet().stream().map(e -> insertMetric(e, "DATA")) + .collect(Collectors.toSet()); + + String branch = "branch_4"; + insertNotMigratedBranch(branch); + String component1 = uuidFactory.create(); + metricsToMigrate.forEach(metricUuid -> insertMeasure(branch, component1, metricUuid, + Map.of("measure_data", "{\"LOW\":3,\"MEDIUM\":0,\"HIGH\":4,\"total\":7}"))); + + underTest.execute(); + + assertBranchMigrated(branch); + assertThat(db.countRowsOfTable("measures")).isEqualTo(1); + + List> measuresFromDB = db.select(format(SELECT_MEASURE, component1)); + assertThat(measuresFromDB).hasSize(1); + + Gson gson = new Gson(); + Map jsonValue = gson.fromJson((String) measuresFromDB.get(0).get("json_value"), Map.class); + + Map expectedExistingMetrics = MeasureMigration.MIGRATION_MAP.keySet().stream().collect( + Collectors.toMap(s -> s, s -> "{\"LOW\":3,\"MEDIUM\":0,\"HIGH\":4,\"total\":7}")); + + Map expectedNewMetrics = MeasureMigration.MIGRATION_MAP.values().stream().collect( + Collectors.toMap(s -> s, s -> 7.0)); + + assertThat(jsonValue).containsAllEntriesOf(expectedExistingMetrics).containsAllEntriesOf(expectedNewMetrics); + } + + @Test + void should_migrate_other_measures_when_there_is_an_error_converting_previous_measures() throws SQLException { + String nclocMetricUuid = insertMetric("ncloc", "INT"); + String maintainabilityMetricUuid = insertMetric(CoreMetrics.MAINTAINABILITY_ISSUES_KEY, "DATA"); + String reliabilityMetricUuid = insertMetric(CoreMetrics.RELIABILITY_ISSUES_KEY, "DATA"); + String securityMetricUuid = insertMetric(CoreMetrics.SECURITY_ISSUES_KEY, "DATA"); + + String branch = "branch_4"; + insertNotMigratedBranch(branch); + String component1 = uuidFactory.create(); + insertMeasure(branch, component1, nclocMetricUuid, Map.of("value", 120)); + // total is not a number + insertMeasure(branch, component1, maintainabilityMetricUuid, Map.of("measure_data", "{\"LOW\":3,\"MEDIUM\":0,\"HIGH\":4," + + "\"total\":\"ABC\"}")); + // total cannot fit in a long + insertMeasure(branch, component1, reliabilityMetricUuid, Map.of("measure_data", "{\"LOW\":3,\"MEDIUM\":0,\"HIGH\":4," + + "\"total\":98723987498723987429874928748748}")); + insertMeasure(branch, component1, securityMetricUuid, Map.of("measure_data", "{\"LOW\":3,\"MEDIUM\":0,\"HIGH\":4,\"total\":37}")); + + logTester.setLevel(Level.DEBUG); + underTest.execute(); + + assertBranchMigrated(branch); + assertThat(db.countRowsOfTable("measures")).isEqualTo(1); + + List> measuresFromDB = db.select(format(SELECT_MEASURE, component1)); + assertThat(measuresFromDB).hasSize(1); + + Gson gson = new Gson(); + Map jsonValue = gson.fromJson((String) measuresFromDB.get(0).get("json_value"), Map.class); + + Map expectedExistingMetrics = Map.of( + "ncloc", 120.0, + CoreMetrics.MAINTAINABILITY_ISSUES_KEY, "{\"LOW\":3,\"MEDIUM\":0,\"HIGH\":4,\"total\":\"ABC\"}", + CoreMetrics.RELIABILITY_ISSUES_KEY, "{\"LOW\":3,\"MEDIUM\":0,\"HIGH\":4,\"total\":98723987498723987429874928748748}", + CoreMetrics.SECURITY_ISSUES_KEY, "{\"LOW\":3,\"MEDIUM\":0,\"HIGH\":4,\"total\":37}" + ); + + Map expectedNewMetrics = Map.of(SoftwareQualitiesMetrics.SOFTWARE_QUALITY_SECURITY_ISSUES_KEY, 37.0); + + assertThat(jsonValue).hasSize(5).containsAllEntriesOf(expectedExistingMetrics).containsAllEntriesOf(expectedNewMetrics); + assertThat(logTester.logs(Level.DEBUG)).contains("Failed to migrate metric reliability_issues with value {\"LOW\":3,\"MEDIUM\":0," + + "\"HIGH\":4,\"total\":98723987498723987429874928748748}"); + } + private void assertBranchMigrated(String branch) { List> result = db.select(format("select %s as \"MIGRATED\" from project_branches where uuid = '%s'", MEASURES_MIGRATED_COLUMN, branch)); assertThat(result) diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/AbstractMigrateLiveMeasuresToMeasures.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/AbstractMigrateLiveMeasuresToMeasures.java index 5ca1074bd78..3421bd0b977 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/AbstractMigrateLiveMeasuresToMeasures.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/AbstractMigrateLiveMeasuresToMeasures.java @@ -207,13 +207,24 @@ public abstract class AbstractMigrateLiveMeasuresToMeasures extends DataChange { Object metricValue = getMetricValue(data, textValue, valueType, numericValue); if (metricValue != null - && !measuresPlannedForDeletion(metricName)) { + && !MeasureMigration.isMetricPlannedForDeletion(metricName)) { measureValues.put(metricName, metricValue); + migrateMeasureIfNeeded(measureValues, metricName, metricValue); } } - private static boolean measuresPlannedForDeletion(String metricName) { - return DeleteSoftwareQualityRatingFromProjectMeasures.SOFTWARE_QUALITY_METRICS_TO_DELETE.contains(metricName); + private static void migrateMeasureIfNeeded(Map measureValues, String metricName, Object metricValue) { + String migratedMetricKey = MeasureMigration.getMigrationMetricKey(metricName); + if (migratedMetricKey != null) { + try { + Object migratedValue = MeasureMigration.migrate(metricValue); + if (migratedValue != null) { + measureValues.put(migratedMetricKey, migratedValue); + } + } catch (Exception e) { + LOGGER.debug("Failed to migrate metric {} with value {}", metricName, metricValue); + } + } } private static Object getMetricValue(@Nullable byte[] data, @Nullable String textValue, String valueType, Double numericValue) { diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/MeasureMigration.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/MeasureMigration.java new file mode 100644 index 00000000000..1ab0ec020b9 --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/MeasureMigration.java @@ -0,0 +1,62 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v108; + +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.CheckForNull; +import org.sonar.api.measures.CoreMetrics; +import org.sonar.core.metric.SoftwareQualitiesMetrics; + +public class MeasureMigration { + + static final Pattern VALUE_EXTRACTION_PATTERN = Pattern.compile("\"total\":(\\d+)"); + + static final Map MIGRATION_MAP = Map.of( + CoreMetrics.MAINTAINABILITY_ISSUES_KEY, SoftwareQualitiesMetrics.SOFTWARE_QUALITY_MAINTAINABILITY_ISSUES_KEY, + CoreMetrics.NEW_MAINTAINABILITY_ISSUES_KEY, SoftwareQualitiesMetrics.NEW_SOFTWARE_QUALITY_MAINTAINABILITY_ISSUES_KEY, + CoreMetrics.RELIABILITY_ISSUES_KEY, SoftwareQualitiesMetrics.SOFTWARE_QUALITY_RELIABILITY_ISSUES_KEY, + CoreMetrics.NEW_RELIABILITY_ISSUES_KEY, SoftwareQualitiesMetrics.NEW_SOFTWARE_QUALITY_RELIABILITY_ISSUES_KEY, + CoreMetrics.SECURITY_ISSUES_KEY, SoftwareQualitiesMetrics.SOFTWARE_QUALITY_SECURITY_ISSUES_KEY, + CoreMetrics.NEW_SECURITY_ISSUES_KEY, SoftwareQualitiesMetrics.NEW_SOFTWARE_QUALITY_SECURITY_ISSUES_KEY); + + private MeasureMigration() { + //Only static methods + } + + @CheckForNull + public static String getMigrationMetricKey(String metricKey) { + return MIGRATION_MAP.get(metricKey); + } + + @CheckForNull + public static Object migrate(Object value) { + Matcher matcher = VALUE_EXTRACTION_PATTERN.matcher(value.toString()); + if (matcher.find()) { + return Long.valueOf(matcher.group(1)); + } + return null; + } + + public static boolean isMetricPlannedForDeletion(String metricKey) { + return DeleteSoftwareQualityRatingFromProjectMeasures.SOFTWARE_QUALITY_METRICS_TO_DELETE.contains(metricKey); + } +} -- 2.39.5