diff options
author | Eric Giffon <eric.giffon@sonarsource.com> | 2024-10-01 18:32:40 +0200 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2024-10-14 20:03:02 +0000 |
commit | 2f2ce0efe6b0c4ed0912b95166e6c2259f3ea3ea (patch) | |
tree | 1a043f50526bad4ef306c727b7e9f745298bce79 | |
parent | fdeb15ab72feed39f26ea32d2c60ffd7a1f3d8e1 (diff) | |
download | sonarqube-2f2ce0efe6b0c4ed0912b95166e6c2259f3ea3ea.tar.gz sonarqube-2f2ce0efe6b0c4ed0912b95166e6c2259f3ea3ea.zip |
SONAR-23213 Measures double write - persist step
31 files changed, 1590 insertions, 3 deletions
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java index ddfa5676e1f..cca8d3ab492 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java @@ -41,6 +41,7 @@ import org.sonar.ce.task.projectanalysis.component.ReferenceBranchComponentUuids import org.sonar.ce.task.projectanalysis.component.ReportModulesPath; import org.sonar.ce.task.projectanalysis.component.SiblingComponentsWithOpenIssues; import org.sonar.ce.task.projectanalysis.component.TreeRootHolderImpl; +import org.sonar.ce.task.projectanalysis.duplication.ComputeDuplicationDataMeasure; import org.sonar.ce.task.projectanalysis.duplication.CrossProjectDuplicationStatusHolderImpl; import org.sonar.ce.task.projectanalysis.duplication.DuplicationMeasures; import org.sonar.ce.task.projectanalysis.duplication.DuplicationRepositoryImpl; @@ -332,6 +333,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop // duplication IntegrateCrossProjectDuplications.class, DuplicationMeasures.class, + ComputeDuplicationDataMeasure.class, // views ViewIndex.class, diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasure.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasure.java new file mode 100644 index 00000000000..7b606c33a58 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasure.java @@ -0,0 +1,92 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.duplication; + +import java.util.Optional; +import org.apache.commons.lang.StringEscapeUtils; +import org.sonar.ce.task.projectanalysis.component.Component; + +import static com.google.common.collect.Iterables.isEmpty; + +/** + * Compute duplication data measures on files, based on the {@link DuplicationRepository} + */ +public class ComputeDuplicationDataMeasure { + private final DuplicationRepository duplicationRepository; + + public ComputeDuplicationDataMeasure(DuplicationRepository duplicationRepository) { + this.duplicationRepository = duplicationRepository; + } + + public Optional<String> compute(Component file) { + Iterable<Duplication> duplications = duplicationRepository.getDuplications(file); + if (isEmpty(duplications)) { + return Optional.empty(); + } + return Optional.of(generateMeasure(file.getKey(), duplications)); + } + + private static String generateMeasure(String componentDbKey, Iterable<Duplication> duplications) { + StringBuilder xml = new StringBuilder(); + xml.append("<duplications>"); + for (Duplication duplication : duplications) { + xml.append("<g>"); + appendDuplication(xml, componentDbKey, duplication.getOriginal(), false); + for (Duplicate duplicate : duplication.getDuplicates()) { + processDuplicationBlock(xml, duplicate, componentDbKey); + } + xml.append("</g>"); + } + xml.append("</duplications>"); + return xml.toString(); + } + + private static void processDuplicationBlock(StringBuilder xml, Duplicate duplicate, String componentDbKey) { + if (duplicate instanceof InnerDuplicate) { + // Duplication is on the same file + appendDuplication(xml, componentDbKey, duplicate); + } else if (duplicate instanceof InExtendedProjectDuplicate inExtendedProjectDuplicate) { + // Duplication is on a different file that is not saved in the DB + appendDuplication(xml, inExtendedProjectDuplicate.getFile().getKey(), duplicate.getTextBlock(), true); + } else if (duplicate instanceof InProjectDuplicate inProjectDuplicate) { + // Duplication is on a different file + appendDuplication(xml, inProjectDuplicate.getFile().getKey(), duplicate); + } else if (duplicate instanceof CrossProjectDuplicate crossProjectDuplicate) { + // Only componentKey is set for cross project duplications + String crossProjectComponentKey = crossProjectDuplicate.getFileKey(); + appendDuplication(xml, crossProjectComponentKey, duplicate); + } else { + throw new IllegalArgumentException("Unsupported type of Duplicate " + duplicate.getClass().getName()); + } + } + + private static void appendDuplication(StringBuilder xml, String componentDbKey, Duplicate duplicate) { + appendDuplication(xml, componentDbKey, duplicate.getTextBlock(), false); + } + + private static void appendDuplication(StringBuilder xml, String componentDbKey, TextBlock textBlock, boolean disableLink) { + int length = textBlock.getEnd() - textBlock.getStart() + 1; + xml.append("<b s=\"").append(textBlock.getStart()) + .append("\" l=\"").append(length) + .append("\" t=\"").append(disableLink) + .append("\" r=\"").append(StringEscapeUtils.escapeXml(componentDbKey)) + .append("\"/>"); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDto.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDto.java index f013e2e6e5f..a518648d223 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDto.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDto.java @@ -60,6 +60,14 @@ public class MeasureToMeasureDto { return out; } + public static Object getMeasureValue(Measure measure) { + Double doubleValue = valueAsDouble(measure); + if (doubleValue != null) { + return doubleValue; + } + return data(measure); + } + private static void setAlert(MeasureDto measureDto, QualityGateStatus qualityGateStatus) { measureDto.setAlertStatus(qualityGateStatus.getStatus().name()); measureDto.setAlertText(qualityGateStatus.getText()); diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistJsonMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistJsonMeasuresStep.java new file mode 100644 index 00000000000..42970e2bea0 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistJsonMeasuresStep.java @@ -0,0 +1,247 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit; +import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolder; +import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter; +import org.sonar.ce.task.projectanalysis.duplication.ComputeDuplicationDataMeasure; +import org.sonar.ce.task.projectanalysis.measure.BestValueOptimization; +import org.sonar.ce.task.projectanalysis.measure.Measure; +import org.sonar.ce.task.projectanalysis.measure.MeasureRepository; +import org.sonar.ce.task.projectanalysis.measure.MeasureToMeasureDto; +import org.sonar.ce.task.projectanalysis.metric.Metric; +import org.sonar.ce.task.projectanalysis.metric.MetricRepository; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.measure.JsonMeasureDto; +import org.sonar.db.measure.JsonMeasureHash; +import org.sonar.db.property.PropertyDto; +import org.springframework.beans.factory.annotation.Autowired; + +import static org.sonar.api.measures.CoreMetrics.DUPLICATIONS_DATA_KEY; +import static org.sonar.api.measures.CoreMetrics.FILE_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.ce.task.projectanalysis.component.Component.Type; +import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER; +import static org.sonar.core.config.CorePropertyDefinitions.SYSTEM_MEASURES_MIGRATION_ENABLED; + +public class PersistJsonMeasuresStep implements ComputationStep { + + /** + * List of metrics that should not be persisted on file measure. + */ + private static final Set<String> NOT_TO_PERSIST_ON_FILE_METRIC_KEYS = Set.of( + FILE_COMPLEXITY_DISTRIBUTION_KEY, + FUNCTION_COMPLEXITY_DISTRIBUTION_KEY); + + // 50 mb + private static final int MAX_TRANSACTION_SIZE = 50_000_000; + private static final Predicate<Measure> NON_EMPTY_MEASURE = measure -> + measure.getValueType() != Measure.ValueType.NO_VALUE || measure.getData() != null; + + private final DbClient dbClient; + private final MetricRepository metricRepository; + private final TreeRootHolder treeRootHolder; + private final MeasureRepository measureRepository; + private final ComputeDuplicationDataMeasure computeDuplicationDataMeasure; + private final int maxTransactionSize; + + @Autowired + public PersistJsonMeasuresStep(DbClient dbClient, MetricRepository metricRepository, TreeRootHolder treeRootHolder, + MeasureRepository measureRepository, @Nullable ComputeDuplicationDataMeasure computeDuplicationDataMeasure) { + this(dbClient, metricRepository, treeRootHolder, measureRepository, computeDuplicationDataMeasure, MAX_TRANSACTION_SIZE); + } + + PersistJsonMeasuresStep(DbClient dbClient, MetricRepository metricRepository, TreeRootHolder treeRootHolder, + MeasureRepository measureRepository, @Nullable ComputeDuplicationDataMeasure computeDuplicationDataMeasure, int maxTransactionSize) { + this.dbClient = dbClient; + this.metricRepository = metricRepository; + this.treeRootHolder = treeRootHolder; + this.measureRepository = measureRepository; + this.computeDuplicationDataMeasure = computeDuplicationDataMeasure; + this.maxTransactionSize = maxTransactionSize; + } + + @Override + public String getDescription() { + return "Persist JSON measures"; + } + + @Override + public void execute(ComputationStep.Context context) { + if (!isMeasuresMigrationEnabled()) { + return; + } + + Component root = treeRootHolder.getRoot(); + CollectComponentsVisitor visitor = new CollectComponentsVisitor(); + new DepthTraversalTypeAwareCrawler(visitor).visit(root); + + Set<JsonMeasureHash> dbMeasureHashes = getDBMeasureHashes(); + Set<String> dbComponents = dbMeasureHashes.stream().map(JsonMeasureHash::componentUuid).collect(Collectors.toSet()); + + List<JsonMeasureDto> inserts = new LinkedList<>(); + List<JsonMeasureDto> updates = new LinkedList<>(); + int insertsOrUpdates = 0; + int unchanged = 0; + int size = 0; + + for (Component component : visitor.components) { + JsonMeasureDto measure = createMeasure(component); + + if (dbMeasureHashes.contains(new JsonMeasureHash(measure.getComponentUuid(), measure.computeJsonValueHash()))) { + unchanged += measure.getMetricValues().size(); + } else { + if (dbComponents.contains(measure.getComponentUuid())) { + updates.add(measure); + } else { + inserts.add(measure); + } + size += measure.getJsonValue().length(); + insertsOrUpdates += measure.getMetricValues().size(); + } + + if (size > maxTransactionSize) { + persist(inserts, updates); + inserts.clear(); + updates.clear(); + size = 0; + } + } + persist(inserts, updates); + updateMeasureMigratedFlag(); + + context.getStatistics() + .add("insertsOrUpdates", insertsOrUpdates) + .add("unchanged", unchanged); + } + + private boolean isMeasuresMigrationEnabled() { + return Optional.ofNullable(dbClient.propertiesDao().selectGlobalProperty(SYSTEM_MEASURES_MIGRATION_ENABLED)) + .map(PropertyDto::getValue) + .map(Boolean::valueOf) + .orElse(false); + } + + private void updateMeasureMigratedFlag() { + Type type = treeRootHolder.getRoot().getType(); + if (type == Type.PROJECT) { + persistBranchFlag(); + } else if (type == Type.VIEW) { + persistPortfolioFlag(); + } + } + + private void persistBranchFlag() { + try (DbSession dbSession = dbClient.openSession(false)) { + dbClient.branchDao().updateMeasuresMigrated(dbSession, treeRootHolder.getRoot().getUuid(), true); + dbSession.commit(); + } + } + + private void persistPortfolioFlag() { + try (DbSession dbSession = dbClient.openSession(false)) { + dbClient.portfolioDao().updateMeasuresMigrated(dbSession, treeRootHolder.getRoot().getUuid(), true); + dbSession.commit(); + } + } + + private Set<JsonMeasureHash> getDBMeasureHashes() { + try (DbSession dbSession = dbClient.openSession(false)) { + return dbClient.jsonMeasureDao().selectBranchMeasureHashes(dbSession, treeRootHolder.getRoot().getUuid()); + } + } + + private JsonMeasureDto createMeasure(Component component) { + JsonMeasureDto measureDto = new JsonMeasureDto(); + measureDto.setComponentUuid(component.getUuid()); + measureDto.setBranchUuid(treeRootHolder.getRoot().getUuid()); + + Map<String, Measure> measures = measureRepository.getRawMeasures(component); + for (Map.Entry<String, Measure> measuresByMetricKey : measures.entrySet()) { + String metricKey = measuresByMetricKey.getKey(); + if (NOT_TO_PERSIST_ON_FILE_METRIC_KEYS.contains(metricKey) && component.getType() == Type.FILE) { + continue; + } + Metric metric = metricRepository.getByKey(metricKey); + Predicate<Measure> notBestValueOptimized = BestValueOptimization.from(metric, component).negate(); + Measure measure = measuresByMetricKey.getValue(); + Stream.of(measure) + .filter(NON_EMPTY_MEASURE) + .filter(notBestValueOptimized) + .map(MeasureToMeasureDto::getMeasureValue) + .filter(Objects::nonNull) + .forEach(value -> measureDto.addValue(metric.getKey(), value)); + } + + if (component.getType() == Type.FILE) { + if (computeDuplicationDataMeasure == null) { + throw new IllegalStateException("ComputeDuplicationDataMeasure not initialized in container"); + } + computeDuplicationDataMeasure.compute(component) + .ifPresent(duplicationData -> measureDto.addValue(DUPLICATIONS_DATA_KEY, duplicationData)); + } + + return measureDto; + } + + private void persist(Collection<JsonMeasureDto> inserts, Collection<JsonMeasureDto> updates) { + if (inserts.isEmpty() && updates.isEmpty()) { + return; + } + try (DbSession dbSession = dbClient.openSession(true)) { + for (JsonMeasureDto m : inserts) { + dbClient.jsonMeasureDao().insert(dbSession, m); + } + for (JsonMeasureDto m : updates) { + dbClient.jsonMeasureDao().update(dbSession, m); + } + dbSession.commit(); + } + } + + private static class CollectComponentsVisitor extends TypeAwareVisitorAdapter { + private final List<Component> components = new LinkedList<>(); + + private CollectComponentsVisitor() { + super(CrawlerDepthLimit.LEAVES, PRE_ORDER); + } + + @Override + public void visitAny(Component component) { + components.add(component); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java index 3fa14e6b7cb..854f78be27d 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java @@ -96,6 +96,7 @@ public class ReportComputationSteps extends AbstractComputationSteps { PersistAnalysisPropertiesStep.class, PersistMeasuresStep.class, PersistLiveMeasuresStep.class, + PersistJsonMeasuresStep.class, PersistDuplicationDataStep.class, PersistAdHocRulesStep.class, PersistIssuesStep.class, diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasureTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasureTest.java new file mode 100644 index 00000000000..1840580f926 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasureTest.java @@ -0,0 +1,112 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.duplication; + +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; +import org.sonar.db.DbTester; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.FILE; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.PROJECT; +import static org.sonar.ce.task.projectanalysis.component.ReportComponent.builder; + +public class ComputeDuplicationDataMeasureTest { + + private static final int ROOT_REF = 1; + private static final String PROJECT_KEY = "PROJECT_KEY"; + private static final String PROJECT_UUID = "u1"; + + private static final int FILE_1_REF = 2; + private static final String FILE_1_KEY = "FILE_1_KEY"; + private static final String FILE_1_UUID = "u2"; + + private static final int FILE_2_REF = 3; + private static final String FILE_2_KEY = "FILE_2_KEY"; + private static final String FILE_2_UUID = "u3"; + + @Rule + public DbTester db = DbTester.create(System2.INSTANCE); + + private final Component file1 = builder(FILE, FILE_1_REF).setKey(FILE_1_KEY).setUuid(FILE_1_UUID).build(); + private final Component file2 = builder(FILE, FILE_2_REF).setKey(FILE_2_KEY).setUuid(FILE_2_UUID).build(); + + @Rule + public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule() + .setRoot( + builder(PROJECT, ROOT_REF).setKey(PROJECT_KEY).setUuid(PROJECT_UUID) + .addChildren(file1, file2) + .build()); + + @Rule + public DuplicationRepositoryRule duplicationRepository = DuplicationRepositoryRule.create(treeRootHolder); + + @Test + public void nothing_to_persist_when_no_duplication() { + assertThat(underTest().compute(file1)).isEmpty(); + } + + @Test + public void compute_duplications_on_same_file() { + duplicationRepository.addDuplication(FILE_1_REF, new TextBlock(1, 5), new TextBlock(6, 10)); + + assertThat(underTest().compute(file1)) + .contains("<duplications><g><b s=\"1\" l=\"5\" t=\"false\" r=\"" + FILE_1_KEY + "\"/><b s=\"6\" l=\"5\" t=\"false\" r=\"" + + FILE_1_KEY + "\"/></g></duplications>"); + } + + @Test + public void compute_duplications_on_different_files() { + duplicationRepository.addDuplication(FILE_1_REF, new TextBlock(1, 5), FILE_2_REF, new TextBlock(6, 10)); + + assertThat(underTest().compute(file1)) + .contains("<duplications><g><b s=\"1\" l=\"5\" t=\"false\" r=\"" + FILE_1_KEY + "\"/><b s=\"6\" l=\"5\" t=\"false\" r=\"" + + FILE_2_KEY + "\"/></g></duplications>"); + assertThat(underTest().compute(file2)).isEmpty(); + } + + @Test + public void compute_duplications_on_unchanged_file() { + duplicationRepository.addExtendedProjectDuplication(FILE_1_REF, new TextBlock(1, 5), FILE_2_REF, new TextBlock(6, 10)); + + assertThat(underTest().compute(file1)) + .contains("<duplications><g><b s=\"1\" l=\"5\" t=\"false\" r=\"" + FILE_1_KEY + "\"/><b s=\"6\" l=\"5\" t=\"true\" r=\"" + + FILE_2_KEY + "\"/></g></duplications>"); + assertThat(underTest().compute(file2)).isEmpty(); + } + + @Test + public void compute_duplications_on_different_projects() { + String fileKeyFromOtherProject = "PROJECT2_KEY:file2"; + duplicationRepository.addCrossProjectDuplication(FILE_1_REF, new TextBlock(1, 5), fileKeyFromOtherProject, new TextBlock(6, 10)); + + assertThat(underTest().compute(file1)) + .contains("<duplications><g><b s=\"1\" l=\"5\" t=\"false\" r=\"" + FILE_1_KEY + "\"/><b s=\"6\" l=\"5\" t=\"false\" r=\"" + + fileKeyFromOtherProject + "\"/></g></duplications>"); + assertThat(underTest().compute(file2)).isEmpty(); + } + + private ComputeDuplicationDataMeasure underTest() { + return new ComputeDuplicationDataMeasure(duplicationRepository); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDtoTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDtoTest.java index e06243d073f..d8f78128887 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDtoTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDtoTest.java @@ -37,6 +37,7 @@ import org.sonar.db.measure.MeasureDto; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.sonar.ce.task.projectanalysis.measure.MeasureToMeasureDto.getMeasureValue; @RunWith(DataProviderRunner.class) public class MeasureToMeasureDtoTest { @@ -180,4 +181,42 @@ public class MeasureToMeasureDtoTest { assertThat(liveMeasureDto.getTextValue()).isEqualTo(Measure.Level.OK.name()); } + + @Test + public void getMeasureValue_returns_null_if_measure_is_empty() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().createNoValue())).isNull(); + } + + @Test + public void getMeasureValue_maps_value_to_1_or_0_and_data_from_data_field_for_BOOLEAN_metric() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().create(true, SOME_DATA))).isEqualTo(1d); + assertThat(getMeasureValue(Measure.newMeasureBuilder().create(false, SOME_DATA))).isEqualTo(0d); + } + + @Test + public void getMeasureValue_maps_value_and_data_from_data_field_for_INT_metric() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().create(123, SOME_DATA))).isEqualTo(123.0); + } + + @Test + public void getMeasureValue_maps_value_and_data_from_data_field_for_LONG_metric() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().create((long) 456, SOME_DATA))).isEqualTo(456.0); + } + + @Test + public void getMeasureValue_maps_value_and_data_from_data_field_for_DOUBLE_metric() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().create(789, 1, SOME_DATA))).isEqualTo(789.0); + } + + @Test + public void getMeasureValue_maps_to_only_data_for_STRING_metric() { + assertThat(getMeasureValue( + Measure.newMeasureBuilder().create(SOME_STRING))).isEqualTo(SOME_STRING); + } + + @Test + public void getMeasureValue_maps_name_of_Level_to_data_and_has_no_value_for_LEVEL_metric() { + assertThat(getMeasureValue( + Measure.newMeasureBuilder().create(Measure.Level.OK))).isEqualTo(Measure.Level.OK.name()); + } } diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistJsonMeasuresStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistJsonMeasuresStepTest.java new file mode 100644 index 00000000000..bbc08fab877 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistJsonMeasuresStepTest.java @@ -0,0 +1,448 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.measures.Metric; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.projectanalysis.analysis.Analysis; +import org.sonar.ce.task.projectanalysis.analysis.MutableAnalysisMetadataHolderRule; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.ReportComponent; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; +import org.sonar.ce.task.projectanalysis.component.ViewsComponent; +import org.sonar.ce.task.projectanalysis.duplication.ComputeDuplicationDataMeasure; +import org.sonar.ce.task.projectanalysis.measure.MeasureRepositoryRule; +import org.sonar.ce.task.projectanalysis.metric.MetricRepositoryRule; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbClient; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; +import org.sonar.db.component.BranchType; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.measure.JsonMeasureDto; +import org.sonar.db.metric.MetricDto; +import org.sonar.server.platform.db.migration.adhoc.AddMeasuresMigratedColumnToPortfoliosTable; +import org.sonar.server.platform.db.migration.adhoc.AddMeasuresMigratedColumnToProjectBranchesTable; +import org.sonar.server.platform.db.migration.adhoc.CreateMeasuresTable; +import org.sonar.server.project.Project; + +import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static java.util.Map.entry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.sonar.api.measures.CoreMetrics.BUGS; +import static org.sonar.api.measures.CoreMetrics.FILE_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.DIRECTORY; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.FILE; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.PROJECT; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.PROJECT_VIEW; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.SUBVIEW; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.VIEW; +import static org.sonar.ce.task.projectanalysis.measure.Measure.newMeasureBuilder; +import static org.sonar.core.config.CorePropertyDefinitions.SYSTEM_MEASURES_MIGRATION_ENABLED; + +public class PersistJsonMeasuresStepTest extends BaseStepTest { + + private static final Metric<?> STRING_METRIC = new Metric.Builder("string-metric", "String metric", Metric.ValueType.STRING).create(); + private static final Metric<?> INT_METRIC = new Metric.Builder("int-metric", "int metric", Metric.ValueType.INT).create(); + private static final Metric<?> METRIC_WITH_BEST_VALUE = new Metric.Builder("best-value-metric", "best value metric", Metric.ValueType.INT) + .setBestValue(0.0) + .setOptimizedBestValue(true) + .create(); + + private static final int REF_1 = 1; + private static final int REF_2 = 2; + private static final int REF_3 = 3; + private static final int REF_4 = 4; + + @Rule + public DbTester db = DbTester.create(System2.INSTANCE); + @Rule + public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule(); + @Rule + public MetricRepositoryRule metricRepository = new MetricRepositoryRule(); + @Rule + public MeasureRepositoryRule measureRepository = MeasureRepositoryRule.create(treeRootHolder, metricRepository); + @Rule + public MutableAnalysisMetadataHolderRule analysisMetadataHolder = new MutableAnalysisMetadataHolderRule(); + private final ComputeDuplicationDataMeasure computeDuplicationDataMeasure = mock(ComputeDuplicationDataMeasure.class); + private final TestComputationStepContext context = new TestComputationStepContext(); + + private final DbClient dbClient = db.getDbClient(); + + @Before + public void setUp() throws SQLException { + new CreateMeasuresTable(db.getDbClient().getDatabase()).execute(); + db.executeDdl("truncate table measures"); + new AddMeasuresMigratedColumnToProjectBranchesTable(db.getDbClient().getDatabase()).execute(); + new AddMeasuresMigratedColumnToPortfoliosTable(db.getDbClient().getDatabase()).execute(); + + db.properties().insertProperty(SYSTEM_MEASURES_MIGRATION_ENABLED, "true", null); + + MetricDto stringMetricDto = + db.measures().insertMetric(m -> m.setKey(STRING_METRIC.getKey()).setValueType(Metric.ValueType.STRING.name())); + MetricDto intMetricDto = db.measures().insertMetric(m -> m.setKey(INT_METRIC.getKey()).setValueType(Metric.ValueType.INT.name())); + MetricDto bestValueMMetricDto = db.measures() + .insertMetric(m -> m.setKey(METRIC_WITH_BEST_VALUE.getKey()).setValueType(Metric.ValueType.INT.name()).setOptimizedBestValue(true).setBestValue(0.0)); + MetricDto bugs = db.measures().insertMetric(m -> m.setKey(BUGS.getKey())); + metricRepository.add(stringMetricDto.getUuid(), STRING_METRIC); + metricRepository.add(intMetricDto.getUuid(), INT_METRIC); + metricRepository.add(bestValueMMetricDto.getUuid(), METRIC_WITH_BEST_VALUE); + metricRepository.add(bugs.getUuid(), BUGS); + } + + @Test + public void description() { + assertThat(step().getDescription()).isEqualTo("Persist JSON measures"); + } + + @Test + public void do_not_process_step_if_measures_migration_disabled() { + prepareProject(); + db.properties().insertProperty(SYSTEM_MEASURES_MIGRATION_ENABLED, "false", null); + + measureRepository.addRawMeasure(REF_4, INT_METRIC.getKey(), newMeasureBuilder().create(123L)); + + step().execute(context); + + assertThat(getBranchMigratedFlag("project-uuid")).isFalse(); + assertThat(db.countRowsOfTable("measures")).isZero(); + } + + @Test + public void persist_live_measures_of_project_analysis() { + prepareProject(); + + // the computed measures + measureRepository.addRawMeasure(REF_1, STRING_METRIC.getKey(), newMeasureBuilder().create("project-value")); + measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("dir-value")); + measureRepository.addRawMeasure(REF_4, STRING_METRIC.getKey(), newMeasureBuilder().create("file-value")); + + step().execute(context); + + // all measures are persisted, from project to file + assertThat(db.countRowsOfTable("measures")).isEqualTo(3); + assertThat(selectMeasure("project-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "project-value")); + assertThat(selectMeasure("dir-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "dir-value")); + assertThat(selectMeasure("file-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "file-value")); + assertThat(getBranchMigratedFlag("project-uuid")).isTrue(); + verifyInsertsOrUpdates(3); + } + + @Test + public void persist_live_measures_of_portfolio_analysis() { + preparePortfolio(); + + // the computed measures + measureRepository.addRawMeasure(REF_1, STRING_METRIC.getKey(), newMeasureBuilder().create("view-value")); + measureRepository.addRawMeasure(REF_2, STRING_METRIC.getKey(), newMeasureBuilder().create("subview-value")); + measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("project-value")); + + step().execute(context); + + assertThat(db.countRowsOfTable("measures")).isEqualTo(3); + assertThat(selectMeasure("view-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "view-value")); + assertThat(selectMeasure("subview-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "subview-value")); + assertThat(selectMeasure("project-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "project-value")); + assertThat(getPortfolioMigratedFlag("view-uuid")).isTrue(); + verifyInsertsOrUpdates(3); + } + + @Test + public void persists_large_number_of_measures() { + int num = 11; + List<ReportComponent> files = new LinkedList<>(); + String valuePrefix = "value".repeat(10); + + for (int i = 0; i < num; i++) { + files.add(ReportComponent.builder(FILE, i).setUuid("file-uuid" + i).build()); + } + Component project = ReportComponent.builder(Component.Type.PROJECT, -1).setUuid("project-uuid") + .addChildren(files.toArray(Component[]::new)) + .build(); + insertBranch(); + treeRootHolder.setRoot(project); + analysisMetadataHolder.setBaseAnalysis(new Analysis.Builder().setUuid("uuid").setCreatedAt(1L).build()); + insertMeasure("file-uuid0", "project-uuid", STRING_METRIC, valuePrefix + "0"); + + for (int i = 0; i < num; i++) { + measureRepository.addRawMeasure(i, STRING_METRIC.getKey(), newMeasureBuilder().create(valuePrefix + i)); + } + db.getSession().commit(); + + PersistJsonMeasuresStep step = new PersistJsonMeasuresStep(dbClient, metricRepository, treeRootHolder, measureRepository, + computeDuplicationDataMeasure, 100); + step.execute(context); + + // all measures are persisted, for project and all files + assertThat(db.countRowsOfTable("measures")).isEqualTo(num + 1); + assertThat(getBranchMigratedFlag("project-uuid")).isTrue(); + verifyInsertsOrUpdates(num - 1); + verifyUnchanged(1); + verify(computeDuplicationDataMeasure, times(num)).compute(any(Component.class)); + } + + @Test + public void do_not_persist_excluded_file_metrics() { + MetricDto fileComplexityMetric = + db.measures().insertMetric(m -> m.setKey(FILE_COMPLEXITY_DISTRIBUTION_KEY).setValueType(Metric.ValueType.STRING.name())); + MetricDto functionComplexityMetric = + db.measures().insertMetric(m -> m.setKey(FUNCTION_COMPLEXITY_DISTRIBUTION_KEY).setValueType(Metric.ValueType.INT.name())); + metricRepository.add(fileComplexityMetric.getUuid(), new Metric.Builder(FILE_COMPLEXITY_DISTRIBUTION_KEY, "File Distribution / " + + "Complexity", + Metric.ValueType.DISTRIB).create()); + metricRepository.add(functionComplexityMetric.getUuid(), new Metric.Builder(FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, "Function " + + "Distribution / Complexity", + Metric.ValueType.DISTRIB).create()); + + prepareProject(); + + // the computed measures + measureRepository.addRawMeasure(REF_1, FILE_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("project-value")); + measureRepository.addRawMeasure(REF_3, FILE_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("dir-value")); + measureRepository.addRawMeasure(REF_4, FILE_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("file-value")); + + measureRepository.addRawMeasure(REF_1, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("project-value")); + measureRepository.addRawMeasure(REF_3, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("dir-value")); + measureRepository.addRawMeasure(REF_4, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("file-value")); + + step().execute(context); + + // all measures are persisted, from project to file + assertThat(db.countRowsOfTable("measures")).isEqualTo(3); + + assertThat(selectMeasure("project-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).contains( + entry(FILE_COMPLEXITY_DISTRIBUTION_KEY, "project-value"), + entry(FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, "project-value"))); + assertThat(selectMeasure("dir-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).contains( + entry(FILE_COMPLEXITY_DISTRIBUTION_KEY, "dir-value"), + entry(FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, "dir-value"))); + assertThat(selectMeasure("file-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()) + .doesNotContainKeys(FILE_COMPLEXITY_DISTRIBUTION_KEY, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY)); + + verifyInsertsOrUpdates(4); + } + + @Test + public void measures_without_value_are_not_persisted() { + prepareProject(); + measureRepository.addRawMeasure(REF_1, STRING_METRIC.getKey(), newMeasureBuilder().createNoValue()); + measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().createNoValue()); + + step().execute(context); + + assertThat(selectMeasure("project-uuid")).hasValueSatisfying(measureDto -> assertThat(measureDto.getMetricValues()).isEmpty()); + verifyInsertsOrUpdates(0); + } + + @Test + public void measures_on_new_code_period_are_persisted() { + prepareProject(); + measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().create(42.0)); + + step().execute(context); + + assertThat(selectMeasure("project-uuid")) + .hasValueSatisfying(persistedMeasure -> assertThat(persistedMeasure.getMetricValues()).containsEntry(INT_METRIC.getKey(), 42.0)); + verifyInsertsOrUpdates(1); + } + + @Test + public void do_not_persist_file_measures_with_best_value() { + prepareProject(); + // measure to be deleted because new value matches the metric best value + insertMeasure("file-uuid", "project-uuid", METRIC_WITH_BEST_VALUE, 123.0); + db.commit(); + + // project measure with metric best value -> persist with value 0 + measureRepository.addRawMeasure(REF_1, METRIC_WITH_BEST_VALUE.getKey(), newMeasureBuilder().create(0)); + // file measure with metric best value -> do not persist + measureRepository.addRawMeasure(REF_4, METRIC_WITH_BEST_VALUE.getKey(), newMeasureBuilder().create(0)); + + step().execute(context); + + assertThatMeasureDoesNotExist("file-uuid", METRIC_WITH_BEST_VALUE.getKey()); + + Optional<JsonMeasureDto> persisted = dbClient.jsonMeasureDao().selectByComponentUuid(db.getSession(), "project-uuid"); + assertThat(persisted).isPresent(); + assertThat(persisted.get().getMetricValues()).containsEntry(METRIC_WITH_BEST_VALUE.getKey(), (double) 0); + + verifyInsertsOrUpdates(1); + } + + @Test + public void do_not_persist_if_value_hash_unchanged() { + prepareProject(); + insertMeasure("file-uuid", "project-uuid", INT_METRIC, 123.0); + db.commit(); + + measureRepository.addRawMeasure(REF_4, INT_METRIC.getKey(), newMeasureBuilder().create(123L)); + + step().execute(context); + + assertThat(getBranchMigratedFlag("project-uuid")).isTrue(); + verifyInsertsOrUpdates(0); + verifyUnchanged(1); + } + + @Test + public void persist_if_value_hash_changed() { + prepareProject(); + insertMeasure("file-uuid", "project-uuid", INT_METRIC, 123.0); + db.commit(); + + measureRepository.addRawMeasure(REF_4, INT_METRIC.getKey(), newMeasureBuilder().create(124L)); + + step().execute(context); + + verifyInsertsOrUpdates(1); + verifyUnchanged(0); + } + + private void insertMeasure(String componentUuid, String projectUuid, Metric<?> metric, Object obj) { + JsonMeasureDto measure = new JsonMeasureDto() + .setComponentUuid(componentUuid) + .setBranchUuid(projectUuid) + .addValue(metric.getKey(), obj); + measure.computeJsonValueHash(); + dbClient.jsonMeasureDao().insert(db.getSession(), measure); + } + + private void assertThatMeasureDoesNotExist(String componentUuid, String metricKey) { + assertThat(dbClient.jsonMeasureDao().selectByComponentUuid(db.getSession(), componentUuid)) + .hasValueSatisfying(measureDto -> assertThat(measureDto.getMetricValues()).doesNotContainKey(metricKey)); + } + + private void prepareProject() { + // tree of components as defined by scanner report + Component project = ReportComponent.builder(PROJECT, REF_1).setUuid("project-uuid") + .addChildren( + ReportComponent.builder(DIRECTORY, REF_3).setUuid("dir-uuid") + .addChildren( + ReportComponent.builder(FILE, REF_4).setUuid("file-uuid") + .build()) + .build()) + .build(); + treeRootHolder.setRoot(project); + analysisMetadataHolder.setProject(new Project(project.getUuid(), project.getKey(), project.getName(), project.getDescription(), + emptyList())); + + // components as persisted in db + insertComponent("project-key", "project-uuid"); + insertComponent("dir-key", "dir-uuid"); + insertComponent("file-key", "file-uuid"); + + // branch is persisted in db + insertBranch(); + } + + private void preparePortfolio() { + // tree of components + Component portfolio = ViewsComponent.builder(VIEW, REF_1).setUuid("view-uuid") + .addChildren( + ViewsComponent.builder(SUBVIEW, REF_2).setUuid("subview-uuid") + .addChildren( + ViewsComponent.builder(PROJECT_VIEW, REF_3).setUuid("project-uuid") + .build()) + .build()) + .build(); + treeRootHolder.setRoot(portfolio); + + // components as persisted in db + ComponentDto portfolioDto = db.components().insertPrivatePortfolio(c -> c.setUuid("view-uuid").setKey("view-key").setBranchUuid("view" + + "-uuid")); + insertComponent("subview-key", "subview-uuid"); + insertComponent("project-key", "project-uuid"); + analysisMetadataHolder.setProject(Project.from(portfolioDto)); + } + + private Optional<JsonMeasureDto> selectMeasure(String componentUuid) { + return dbClient.jsonMeasureDao().selectByComponentUuid(db.getSession(), componentUuid); + } + + private void insertBranch() { + dbClient.branchDao().insert(db.getSession(), new BranchDto().setUuid("project-uuid").setProjectUuid("project-uuid").setKey("branch") + .setBranchType(BranchType.BRANCH)); + db.commit(); + } + + private ComponentDto insertComponent(String key, String uuid) { + ComponentDto componentDto = new ComponentDto() + .setKey(key) + .setUuid(uuid) + .setUuidPath(uuid + ".") + .setRootUuid(uuid) + .setBranchUuid(uuid); + db.components().insertComponent(componentDto); + return componentDto; + } + + @Override + protected ComputationStep step() { + return new PersistJsonMeasuresStep(dbClient, metricRepository, treeRootHolder, measureRepository, computeDuplicationDataMeasure); + } + + private boolean getBranchMigratedFlag(String branch) { + List<Map<String, Object>> result = db.select(format("select measures_migrated as \"MIGRATED\" from project_branches where uuid = '%s'", + branch)); + assertThat(result).hasSize(1); + + return (boolean) result.get(0).get("MIGRATED"); + } + + private boolean getPortfolioMigratedFlag(String portfolio) { + List<Map<String, Object>> result = db.select(format("select measures_migrated as \"MIGRATED\" from portfolios where uuid = '%s'", + portfolio)); + assertThat(result).hasSize(1); + + return (boolean) result.get(0).get("MIGRATED"); + } + + private void verifyInsertsOrUpdates(int expectedInsertsOrUpdates) { + context.getStatistics().assertValue("insertsOrUpdates", expectedInsertsOrUpdates); + } + + private void verifyUnchanged(int expectedUnchanged) { + context.getStatistics().assertValue("unchanged", expectedUnchanged); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java index 3940a4a4023..28ed0da32fd 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java @@ -45,6 +45,7 @@ import org.sonar.db.event.EventDao; import org.sonar.db.issue.IssueChangeDao; import org.sonar.db.issue.IssueDao; import org.sonar.db.mapping.ProjectMappingsDao; +import org.sonar.db.measure.JsonMeasureDao; import org.sonar.db.measure.LiveMeasureDao; import org.sonar.db.measure.MeasureDao; import org.sonar.db.metric.MetricDao; @@ -131,6 +132,7 @@ public class DaoModule extends Module { IssueChangeDao.class, IssueDao.class, LiveMeasureDao.class, + JsonMeasureDao.class, MeasureDao.class, MetricDao.class, NewCodePeriodDao.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java index c4168294b63..d5e1d2ce396 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java @@ -45,6 +45,7 @@ import org.sonar.db.event.EventDao; import org.sonar.db.issue.IssueChangeDao; import org.sonar.db.issue.IssueDao; import org.sonar.db.mapping.ProjectMappingsDao; +import org.sonar.db.measure.JsonMeasureDao; import org.sonar.db.measure.LiveMeasureDao; import org.sonar.db.measure.MeasureDao; import org.sonar.db.metric.MetricDao; @@ -161,6 +162,7 @@ public class DbClient { private final QProfileEditUsersDao qProfileEditUsersDao; private final QProfileEditGroupsDao qProfileEditGroupsDao; private final LiveMeasureDao liveMeasureDao; + private final JsonMeasureDao jsonMeasureDao; private final WebhookDao webhookDao; private final WebhookDeliveryDao webhookDeliveryDao; private final ProjectMappingsDao projectMappingsDao; @@ -198,6 +200,7 @@ public class DbClient { componentDao = getDao(map, ComponentDao.class); componentKeyUpdaterDao = getDao(map, ComponentKeyUpdaterDao.class); measureDao = getDao(map, MeasureDao.class); + jsonMeasureDao = getDao(map, JsonMeasureDao.class); userDao = getDao(map, UserDao.class); userGroupDao = getDao(map, UserGroupDao.class); userTokenDao = getDao(map, UserTokenDao.class); @@ -515,6 +518,10 @@ public class DbClient { return liveMeasureDao; } + public JsonMeasureDao jsonMeasureDao() { + return jsonMeasureDao; + } + protected <K extends Dao> K getDao(Map<Class, Dao> map, Class<K> clazz) { return (K) map.get(clazz); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java index 0b5f4d8e236..86fcb13b026 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java @@ -77,6 +77,7 @@ import org.sonar.db.issue.NewCodeReferenceIssueDto; import org.sonar.db.issue.PrIssueDto; import org.sonar.db.mapping.ProjectMappingDto; import org.sonar.db.mapping.ProjectMappingsMapper; +import org.sonar.db.measure.JsonMeasureMapper; import org.sonar.db.measure.LargestBranchNclocDto; import org.sonar.db.measure.LiveMeasureMapper; import org.sonar.db.measure.MeasureDto; @@ -265,6 +266,7 @@ public class MyBatis { ComponentKeyUpdaterMapper.class, ComponentMapper.class, LiveMeasureMapper.class, + JsonMeasureMapper.class, DefaultQProfileMapper.class, DuplicationMapper.class, EsQueueMapper.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java index 5d4b6d3696f..9bc71dc3534 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java @@ -177,4 +177,9 @@ public class BranchDao implements Dao { } return false; } + + public long updateMeasuresMigrated(DbSession dbSession, String branchUuid, boolean measuresMigrated) { + long now = system2.now(); + return mapper(dbSession).updateMeasuresMigrated(branchUuid, measuresMigrated, now); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java index bd1c299bd5d..24cdbacc68c 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java @@ -23,7 +23,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; -import javax.annotation.Nullable; import org.apache.ibatis.annotations.Param; public interface BranchMapper { @@ -73,4 +72,5 @@ public interface BranchMapper { short doAnyOfComponentsNeedIssueSync(@Param("componentKeys") List<String> components); + int updateMeasuresMigrated(@Param("uuid") String uuid, @Param("measuresMigrated") boolean measuresMigrated, @Param("now") long now); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureDao.java new file mode 100644 index 00000000000..49b9aa77b8b --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureDao.java @@ -0,0 +1,55 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.Optional; +import java.util.Set; +import org.sonar.api.utils.System2; +import org.sonar.db.Dao; +import org.sonar.db.DbSession; + +public class JsonMeasureDao implements Dao { + + private final System2 system2; + + public JsonMeasureDao(System2 system2) { + this.system2 = system2; + } + + public int insert(DbSession dbSession, JsonMeasureDto dto) { + return mapper(dbSession).insert(dto, system2.now()); + } + + public int update(DbSession dbSession, JsonMeasureDto dto) { + return mapper(dbSession).update(dto, system2.now()); + } + + public Optional<JsonMeasureDto> selectByComponentUuid(DbSession dbSession, String componentUuid) { + return Optional.ofNullable(mapper(dbSession).selectByComponentUuid(componentUuid)); + } + + public Set<JsonMeasureHash> selectBranchMeasureHashes(DbSession dbSession, String branchUuid) { + return mapper(dbSession).selectBranchMeasureHashes(branchUuid); + } + + private static JsonMeasureMapper mapper(DbSession dbSession) { + return dbSession.getMapper(JsonMeasureMapper.class); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureDto.java new file mode 100644 index 00000000000..7cc4c95f06a --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureDto.java @@ -0,0 +1,101 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.util.Map; +import java.util.TreeMap; +import org.apache.commons.codec.digest.MurmurHash3; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class JsonMeasureDto { + + private static final Gson GSON = new Gson(); + + private String componentUuid; + private String branchUuid; + // measures are kept sorted by metric key so that the value hash is consistent + private Map<String, Object> metricValues = new TreeMap<>(); + private Long jsonValueHash; + + public JsonMeasureDto() { + // empty constructor + } + + public String getComponentUuid() { + return componentUuid; + } + + public JsonMeasureDto setComponentUuid(String s) { + this.componentUuid = s; + return this; + } + + public String getBranchUuid() { + return branchUuid; + } + + public JsonMeasureDto setBranchUuid(String s) { + this.branchUuid = s; + return this; + } + + public Map<String, Object> getMetricValues() { + return metricValues; + } + + public JsonMeasureDto addValue(String metricKey, Object value) { + metricValues.put(metricKey, value); + return this; + } + + // used by MyBatis mapper + public String getJsonValue() { + return GSON.toJson(metricValues); + } + + // used by MyBatis mapper + public JsonMeasureDto setJsonValue(String jsonValue) { + metricValues = GSON.fromJson(jsonValue, new TypeToken<TreeMap<String, Object>>() { + }.getType()); + return this; + } + + public Long getJsonValueHash() { + return jsonValueHash; + } + + public long computeJsonValueHash() { + jsonValueHash = MurmurHash3.hash128(getJsonValue().getBytes(UTF_8))[0]; + return jsonValueHash; + } + + @Override + public String toString() { + return "JsonMeasureDto{" + + "componentUuid='" + componentUuid + '\'' + + ", branchUuid='" + branchUuid + '\'' + + ", metricValues=" + metricValues + + ", jsonValueHash=" + jsonValueHash + + '}'; + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureHash.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureHash.java new file mode 100644 index 00000000000..4daffcc7164 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureHash.java @@ -0,0 +1,32 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.Comparator; + +public record JsonMeasureHash(String componentUuid, Long jsonValueHash) implements Comparable<JsonMeasureHash> { + + @Override + public int compareTo(JsonMeasureHash o) { + return Comparator.comparing(JsonMeasureHash::componentUuid) + .thenComparing(JsonMeasureHash::jsonValueHash) + .compare(this, o); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureMapper.java new file mode 100644 index 00000000000..75db1b79ad5 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/JsonMeasureMapper.java @@ -0,0 +1,40 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.Set; +import javax.annotation.CheckForNull; +import org.apache.ibatis.annotations.Param; + +public interface JsonMeasureMapper { + + int insert( + @Param("dto") JsonMeasureDto dto, + @Param("now") long now); + + int update( + @Param("dto") JsonMeasureDto dto, + @Param("now") long now); + + @CheckForNull + JsonMeasureDto selectByComponentUuid(@Param("componentUuid") String componentUuid); + + Set<JsonMeasureHash> selectBranchMeasureHashes(@Param("branchUuid") String branchUuid); +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java index 5d8689db6bb..cbcfd25b3c6 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java @@ -268,4 +268,8 @@ public class PortfolioDao implements Dao { return portfolioDto.isRoot() ? Qualifiers.VIEW : Qualifiers.SUBVIEW; } + public long updateMeasuresMigrated(DbSession dbSession, String branchUuid, boolean measuresMigrated) { + long now = system2.now(); + return mapper(dbSession).updateMeasuresMigrated(branchUuid, measuresMigrated, now); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java index 249aec0e286..2e273acc6d7 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java @@ -111,4 +111,6 @@ public interface PortfolioMapper { List<ReferenceDto> selectAllReferencesToApplicationsInHierarchy(String rootUuid); List<PortfolioDto> selectRootOfReferencersToAppBranch(@Param("appUuid") String appUuid, @Param("appBranchKey") String appBranchKey); + + int updateMeasuresMigrated(@Param("uuid") String uuid, @Param("measuresMigrated") boolean measuresMigrated, @Param("now") long now); } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml index 4f981d9aafe..eafb0b008cb 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml @@ -248,4 +248,12 @@ from dual </select> + <update id="updateMeasuresMigrated"> + update project_branches + set + measures_migrated = #{measuresMigrated, jdbcType=BOOLEAN}, + updated_at = #{now, jdbcType=BIGINT} + where + uuid = #{uuid, jdbcType=VARCHAR} + </update> </mapper> diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/JsonMeasureMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/JsonMeasureMapper.xml new file mode 100644 index 00000000000..c880ec21228 --- /dev/null +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/JsonMeasureMapper.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "mybatis-3-mapper.dtd"> + +<mapper namespace="org.sonar.db.measure.JsonMeasureMapper"> + + <insert id="insert" parameterType="map" useGeneratedKeys="false"> + insert into measures ( + component_uuid, + branch_uuid, + json_value, + json_value_hash, + created_at, + updated_at + ) values ( + #{dto.componentUuid, jdbcType=VARCHAR}, + #{dto.branchUuid, jdbcType=VARCHAR}, + #{dto.jsonValue, jdbcType=VARCHAR}, + #{dto.jsonValueHash, jdbcType=BIGINT}, + #{now, jdbcType=BIGINT}, + #{now, jdbcType=BIGINT} + ) + </insert> + + <update id="update" parameterType="map" useGeneratedKeys="false"> + update measures set + json_value = #{dto.jsonValue, jdbcType=VARCHAR}, + json_value_hash = #{dto.jsonValueHash, jdbcType=BIGINT}, + updated_at = #{now, jdbcType=BIGINT} + where component_uuid = #{dto.componentUuid, jdbcType=VARCHAR} + </update> + + <sql id="columns"> + m.component_uuid as componentUuid, + m.branch_uuid as branchUuid, + m.json_value as jsonValue, + m.json_value_hash as jsonValueHash + </sql> + + <select id="selectByComponentUuid" parameterType="map" resultType="org.sonar.db.measure.JsonMeasureDto"> + select + <include refid="columns"/> + from measures m + where + m.component_uuid = #{componentUuid, jdbcType=VARCHAR} + </select> + + <select id="selectBranchMeasureHashes" resultType="org.sonar.db.measure.JsonMeasureHash"> + select component_uuid, json_value_hash + from measures + where branch_uuid = #{branchUuid, jdbcType=VARCHAR} + </select> + +</mapper> diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml index be38d220f1b..832fc96bb4a 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml @@ -537,4 +537,13 @@ WHERE uuid = #{uuid,jdbcType=VARCHAR} </update> + + <update id="updateMeasuresMigrated"> + update portfolios + set + measures_migrated = #{measuresMigrated, jdbcType=BOOLEAN}, + updated_at = #{now, jdbcType=BIGINT} + where + uuid = #{uuid, jdbcType=VARCHAR} + </update> </mapper> diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java index 4f1fc34b692..8f42fc67056 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java @@ -22,6 +22,7 @@ package org.sonar.db.component; import com.google.common.collect.Sets; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import java.sql.SQLException; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -38,10 +39,13 @@ import org.sonar.api.impl.utils.TestSystem2; import org.sonar.api.utils.System2; import org.sonar.db.DbSession; import org.sonar.db.DbTester; +import org.sonar.db.dialect.Oracle; import org.sonar.db.metric.MetricDto; import org.sonar.db.project.ProjectDto; import org.sonar.db.protobuf.DbProjectBranches; +import org.sonar.server.platform.db.migration.adhoc.AddMeasuresMigratedColumnToProjectBranchesTable; +import static java.lang.String.format; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; @@ -776,4 +780,31 @@ public class BranchDaoTest { assertThat(underTest.doAnyOfComponentsNeedIssueSync(dbSession, componentKeys)).isTrue(); } + + @Test + public void updateMeasuresMigrated() throws SQLException { + new AddMeasuresMigratedColumnToProjectBranchesTable(db.getDbClient().getDatabase()).execute(); + + ComponentDto project = db.components().insertPrivateProject(); + String uuid1 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH)).uuid(); + String uuid2 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH)).uuid(); + + underTest.updateMeasuresMigrated(dbSession, uuid1, true); + underTest.updateMeasuresMigrated(dbSession, uuid2, false); + + assertThat(getMeasuresMigrated(uuid1)).isTrue(); + assertThat(getMeasuresMigrated(uuid2)).isFalse(); + } + + private boolean getMeasuresMigrated(String uuid1) { + List<Map<String, Object>> select = db.select(dbSession, + format("select measures_migrated as \"MIGRATED\" from project_branches where uuid = '%s'", uuid1)); + + assertThat(select).hasSize(1); + Object value = select.get(0).get("MIGRATED"); + if (db.getDbClient().getDatabase().getDialect().getId().equals(Oracle.ID)) { + return (long) value == 1; + } + return (boolean) value; + } } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/JsonMeasureDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/JsonMeasureDaoTest.java new file mode 100644 index 00000000000..f0b6d913c6b --- /dev/null +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/JsonMeasureDaoTest.java @@ -0,0 +1,122 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.db.DbTester; +import org.sonar.server.platform.db.migration.adhoc.AddMeasuresMigratedColumnToPortfoliosTable; +import org.sonar.server.platform.db.migration.adhoc.AddMeasuresMigratedColumnToProjectBranchesTable; +import org.sonar.server.platform.db.migration.adhoc.CreateIndexOnPortfoliosMeasuresMigrated; +import org.sonar.server.platform.db.migration.adhoc.CreateIndexOnProjectBranchesMeasuresMigrated; +import org.sonar.server.platform.db.migration.adhoc.CreateMeasuresTable; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.db.measure.MeasureTesting.newJsonMeasure; + +public class JsonMeasureDaoTest { + + @Rule + public final DbTester db = DbTester.create(System2.INSTANCE); + + private final JsonMeasureDao underTest = db.getDbClient().jsonMeasureDao(); + + @Before + public void setUp() throws Exception { + new CreateMeasuresTable(db.getDbClient().getDatabase()).execute(); + new AddMeasuresMigratedColumnToProjectBranchesTable(db.getDbClient().getDatabase()).execute(); + new AddMeasuresMigratedColumnToPortfoliosTable(db.getDbClient().getDatabase()).execute(); + new CreateIndexOnProjectBranchesMeasuresMigrated(db.getDbClient().getDatabase()).execute(); + new CreateIndexOnPortfoliosMeasuresMigrated(db.getDbClient().getDatabase()).execute(); + } + + @Test + public void insert_measure() { + JsonMeasureDto dto = newJsonMeasure(); + int count = underTest.insert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyTableSize(1); + verifyPersisted(dto); + } + + @Test + public void update_measure() { + JsonMeasureDto dto = newJsonMeasure(); + underTest.insert(db.getSession(), dto); + + dto.addValue("metric1", "value1"); + dto.computeJsonValueHash(); + int count = underTest.update(db.getSession(), dto); + + assertThat(count).isEqualTo(1); + verifyTableSize(1); + verifyPersisted(dto); + } + + @Test + public void select_measure() { + JsonMeasureDto measure1 = newJsonMeasure(); + JsonMeasureDto measure2 = newJsonMeasure(); + underTest.insert(db.getSession(), measure1); + underTest.insert(db.getSession(), measure2); + + assertThat(underTest.selectByComponentUuid(db.getSession(), measure1.getComponentUuid())) + .hasValueSatisfying(selected -> assertThat(selected).usingRecursiveComparison().isEqualTo(measure1)); + assertThat(underTest.selectByComponentUuid(db.getSession(), "unknown-component")).isEmpty(); + } + + @Test + public void select_branch_measure_hashes() { + JsonMeasureDto measure1 = new JsonMeasureDto() + .setComponentUuid("c1") + .setBranchUuid("b1") + .addValue("metric1", "value1"); + JsonMeasureDto measure2 = new JsonMeasureDto() + .setComponentUuid("c2") + .setBranchUuid("b1") + .addValue("metric2", "value2"); + JsonMeasureDto measure3 = new JsonMeasureDto() + .setComponentUuid("c3") + .setBranchUuid("b3") + .addValue("metric3", "value3"); + long hash1 = measure1.computeJsonValueHash(); + long hash2 = measure2.computeJsonValueHash(); + measure3.computeJsonValueHash(); + + underTest.insert(db.getSession(), measure1); + underTest.insert(db.getSession(), measure2); + underTest.insert(db.getSession(), measure3); + + assertThat(underTest.selectBranchMeasureHashes(db.getSession(), "b1")) + .containsOnly(new JsonMeasureHash("c1", hash1), new JsonMeasureHash("c2", hash2)); + } + + private void verifyTableSize(int expectedSize) { + assertThat(db.countRowsOfTable(db.getSession(), "measures")).isEqualTo(expectedSize); + } + + private void verifyPersisted(JsonMeasureDto dto) { + assertThat(underTest.selectByComponentUuid(db.getSession(), dto.getComponentUuid())).hasValueSatisfying(selected -> { + assertThat(selected).usingRecursiveComparison().isEqualTo(dto); + }); + } +} diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/JsonMeasureDtoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/JsonMeasureDtoTest.java new file mode 100644 index 00000000000..c953537e458 --- /dev/null +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/JsonMeasureDtoTest.java @@ -0,0 +1,68 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.Map; +import java.util.TreeMap; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JsonMeasureDtoTest { + + @Test + public void compute_json_value_hash() { + JsonMeasureDto measureDto = new JsonMeasureDto(); + measureDto.setJsonValue("{\"key\":\"value\"}"); + assertThat(measureDto.getJsonValueHash()).isNull(); + assertThat(measureDto.computeJsonValueHash()).isEqualTo(2887272982314571750L); + assertThat(measureDto.getJsonValueHash()).isEqualTo(2887272982314571750L); + } + + @Test + public void getMetricValues_returns_all_values_ordered() { + JsonMeasureDto measureDto = new JsonMeasureDto() + .addValue("string-metric", "value") + .addValue("int-metric", 1) + .addValue("long-metric", 2L); + assertThat(measureDto.getMetricValues()).containsExactlyEntriesOf(new TreeMap<>(Map.of( + "int-metric", 1, + "long-metric", 2L, + "string-metric", "value" + ))); + } + + @Test + public void toString_prints_all_fields() { + JsonMeasureDto measureDto = new JsonMeasureDto() + .setBranchUuid("branch-uuid") + .setComponentUuid("component-uuid") + .addValue("int-metric", 12) + .addValue("double-metric", 34.5) + .addValue("boolean-metric", true) + .addValue("string-metric", "value"); + measureDto.computeJsonValueHash(); + + assertThat(measureDto).hasToString("JsonMeasureDto{componentUuid='component-uuid'" + + ", branchUuid='branch-uuid'" + + ", metricValues={boolean-metric=true, double-metric=34.5, int-metric=12, string-metric=value}" + + ", jsonValueHash=-1071134275520515337}"); + } +} diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/JsonMeasureHashTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/JsonMeasureHashTest.java new file mode 100644 index 00000000000..05e14cf1e1f --- /dev/null +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/JsonMeasureHashTest.java @@ -0,0 +1,54 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.List; +import java.util.TreeSet; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JsonMeasureHashTest { + + @Test + public void hashCode_depends_on_both_fields() { + JsonMeasureHash measureHash1 = new JsonMeasureHash("component1", 123L); + JsonMeasureHash measureHash2 = new JsonMeasureHash("component", 123L); + JsonMeasureHash measureHash3 = new JsonMeasureHash("component", 124L); + + assertThat(measureHash1) + .doesNotHaveSameHashCodeAs(measureHash2) + .doesNotHaveSameHashCodeAs(measureHash3) + .isNotEqualTo(measureHash2) + .isNotEqualTo(measureHash3); + } + + @Test + public void sort_by_component_and_hash() { + JsonMeasureHash measureHash1 = new JsonMeasureHash("A", 1L); + JsonMeasureHash measureHash2 = new JsonMeasureHash("A", 2L); + JsonMeasureHash measureHash3 = new JsonMeasureHash("B", 1L); + JsonMeasureHash measureHash4 = new JsonMeasureHash("B", 2L); + + TreeSet<JsonMeasureHash> set = new TreeSet<>(List.of(measureHash1, measureHash2, measureHash3, measureHash4)); + + assertThat(set).containsExactly(measureHash1, measureHash2, measureHash3, measureHash4); + } +} diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java index 605dd22d657..ae12b1b059d 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java @@ -19,6 +19,9 @@ */ package org.sonar.db.portfolio; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; import java.util.Set; import org.junit.Rule; import org.junit.Test; @@ -29,9 +32,12 @@ import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.audit.AuditPersister; import org.sonar.db.component.BranchDto; +import org.sonar.db.dialect.Oracle; import org.sonar.db.project.ApplicationProjectDto; import org.sonar.db.project.ProjectDto; +import org.sonar.server.platform.db.migration.adhoc.AddMeasuresMigratedColumnToPortfoliosTable; +import static java.lang.String.format; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; import static org.assertj.core.api.Assertions.assertThat; @@ -699,6 +705,32 @@ public class PortfolioDaoTest { assertThat(db.countRowsOfTable(session, "portfolio_proj_branches")).isZero(); } + @Test + public void update_measures_migrated() throws SQLException { + new AddMeasuresMigratedColumnToPortfoliosTable(db.getDbClient().getDatabase()).execute(); + + PortfolioDto portfolio1 = db.components().insertPrivatePortfolioDto("name1"); + PortfolioDto portfolio2 = db.components().insertPrivatePortfolioDto("name2"); + + portfolioDao.updateMeasuresMigrated(session, portfolio1.getUuid(), true); + portfolioDao.updateMeasuresMigrated(session, portfolio2.getUuid(), false); + + assertThat(getMeasuresMigrated(portfolio1.getUuid())).isTrue(); + assertThat(getMeasuresMigrated(portfolio2.getUuid())).isFalse(); + } + + private boolean getMeasuresMigrated(String uuid1) { + List<Map<String, Object>> select = db.select(session, + format("select measures_migrated as \"MIGRATED\" from portfolios where uuid = '%s'", uuid1)); + + assertThat(select).hasSize(1); + Object value = select.get(0).get("MIGRATED"); + if (db.getDbClient().getDatabase().getDialect().getId().equals(Oracle.ID)) { + return (long) value == 1; + } + return (boolean) value; + } + private PortfolioDto addPortfolio(PortfolioDto parent) { return addPortfolio(parent, UuidFactoryFast.getInstance().create()); } diff --git a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureTesting.java b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureTesting.java index 675aa941777..d86c390ba68 100644 --- a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureTesting.java +++ b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureTesting.java @@ -80,6 +80,15 @@ public class MeasureTesting { .setValue((double) cursor++); } + public static JsonMeasureDto newJsonMeasure() { + JsonMeasureDto measureDto = new JsonMeasureDto() + .setComponentUuid(String.valueOf(cursor++)) + .setBranchUuid(String.valueOf(cursor++)) + .addValue("metric" + cursor++, (double) cursor++); + measureDto.computeJsonValueHash(); + return measureDto; + } + public static LiveMeasureDto createLiveMeasure(MetricDto metricDto, ComponentDto componentDto) { BiConsumer<MetricDto, MeasureAdapter> populator = specificLiveMeasurePopulator.getOrDefault(metricDto.getKey(), defaultLiveMeasurePopulator); LiveMeasureDto liveMeasureDto = newLiveMeasure(componentDto, metricDto); diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/platform/ws/PrepareMigrationAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/platform/ws/PrepareMigrationAction.java index a89d1eb8c1d..bfb5ccda0a2 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/platform/ws/PrepareMigrationAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/platform/ws/PrepareMigrationAction.java @@ -36,6 +36,7 @@ import org.sonar.server.platform.db.migration.adhoc.CreateMeasuresTable; import org.sonar.server.user.UserSession; import static java.lang.String.format; +import static org.sonar.core.config.CorePropertyDefinitions.SYSTEM_MEASURES_MIGRATION_ENABLED; /** * Implementation of the {@code prepare_migration} action for the System WebService. @@ -43,7 +44,6 @@ import static java.lang.String.format; public class PrepareMigrationAction implements SystemWsAction { public static final String PARAM_ENABLE = "enable"; - public static final String SYSTEM_MEASURES_MIGRATION_ENABLED = "system.measures.migration.enabled"; private final UserSession userSession; private final DbClient dbClient; private final CreateMeasuresTable createMeasuresTable; diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/platform/ws/PrepareMigrationActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/platform/ws/PrepareMigrationActionTest.java index 1b6de61b1d5..11bc6c69226 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/platform/ws/PrepareMigrationActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/platform/ws/PrepareMigrationActionTest.java @@ -45,7 +45,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; -import static org.sonar.server.platform.ws.PrepareMigrationAction.SYSTEM_MEASURES_MIGRATION_ENABLED; +import static org.sonar.core.config.CorePropertyDefinitions.SYSTEM_MEASURES_MIGRATION_ENABLED; import static org.sonar.test.JsonAssert.assertJson; @RunWith(DataProviderRunner.class) diff --git a/sonar-core/src/main/java/org/sonar/core/config/CorePropertyDefinitions.java b/sonar-core/src/main/java/org/sonar/core/config/CorePropertyDefinitions.java index 4d3b1d3deb5..318a4f0cf8a 100644 --- a/sonar-core/src/main/java/org/sonar/core/config/CorePropertyDefinitions.java +++ b/sonar-core/src/main/java/org/sonar/core/config/CorePropertyDefinitions.java @@ -46,6 +46,8 @@ public class CorePropertyDefinitions { public static final String PLUGINS_RISK_CONSENT = "sonar.plugins.risk.consent"; public static final String SUBCATEGORY_PROJECT_CREATION = "subProjectCreation"; + public static final String SYSTEM_MEASURES_MIGRATION_ENABLED = "system.measures.migration.enabled"; + private CorePropertyDefinitions() { // only static stuff } |