diff options
author | Eric Giffon <eric.giffon@sonarsource.com> | 2024-08-29 14:37:24 +0200 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2024-10-09 20:02:46 +0000 |
commit | 264769a69d03fd61c87491be3da5f101469bb60c (patch) | |
tree | ac39fd8068d8155ddb93e306fa542651bcb5facd | |
parent | a4da413ce8b4eab78aadc9e492d437f45d9b7e6f (diff) | |
download | sonarqube-264769a69d03fd61c87491be3da5f101469bb60c.tar.gz sonarqube-264769a69d03fd61c87491be3da5f101469bb60c.zip |
SONAR-22872 CE step to persist measures in JSON format
21 files changed, 1395 insertions, 2 deletions
diff --git a/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistMeasuresStepTest.java b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistMeasuresStepTest.java new file mode 100644 index 00000000000..6c7ac0e1b52 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistMeasuresStepTest.java @@ -0,0 +1,382 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.sonar.api.measures.Metric; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.projectanalysis.analysis.Analysis; +import org.sonar.ce.task.projectanalysis.analysis.MutableAnalysisMetadataHolderRule; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.ReportComponent; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; +import org.sonar.ce.task.projectanalysis.component.ViewsComponent; +import org.sonar.ce.task.projectanalysis.duplication.ComputeDuplicationDataMeasure; +import org.sonar.ce.task.projectanalysis.measure.MeasureRepositoryRule; +import org.sonar.ce.task.projectanalysis.metric.MetricRepositoryRule; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbClient; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.measure.MeasureDto; +import org.sonar.db.metric.MetricDto; +import org.sonar.server.project.Project; + +import static java.util.Collections.emptyList; +import static java.util.Map.entry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.sonar.api.measures.CoreMetrics.FILE_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.DIRECTORY; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.FILE; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.PROJECT; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.PROJECT_VIEW; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.SUBVIEW; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.VIEW; +import static org.sonar.ce.task.projectanalysis.measure.Measure.newMeasureBuilder; + +class PersistMeasuresStepTest { + + private static final Metric<?> STRING_METRIC = new Metric.Builder("string-metric", "String metric", Metric.ValueType.STRING).create(); + private static final Metric<?> INT_METRIC = new Metric.Builder("int-metric", "int metric", Metric.ValueType.INT).create(); + private static final Metric<?> METRIC_WITH_BEST_VALUE = new Metric.Builder("best-value-metric", "best value metric", Metric.ValueType.INT) + .setBestValue(0.0) + .setOptimizedBestValue(true) + .create(); + + private static final int REF_1 = 1; + private static final int REF_2 = 2; + private static final int REF_3 = 3; + private static final int REF_4 = 4; + + @RegisterExtension + public DbTester db = DbTester.create(System2.INSTANCE); + @RegisterExtension + public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule(); + @RegisterExtension + public MetricRepositoryRule metricRepository = new MetricRepositoryRule(); + @RegisterExtension + public MeasureRepositoryRule measureRepository = MeasureRepositoryRule.create(treeRootHolder, metricRepository); + @RegisterExtension + public MutableAnalysisMetadataHolderRule analysisMetadataHolder = new MutableAnalysisMetadataHolderRule(); + private final ComputeDuplicationDataMeasure computeDuplicationDataMeasure = mock(ComputeDuplicationDataMeasure.class); + private final TestComputationStepContext context = new TestComputationStepContext(); + + private final DbClient dbClient = db.getDbClient(); + + @BeforeEach + public void setUp() { + MetricDto stringMetricDto = + db.measures().insertMetric(m -> m.setKey(STRING_METRIC.getKey()).setValueType(Metric.ValueType.STRING.name())); + MetricDto intMetricDto = db.measures().insertMetric(m -> m.setKey(INT_METRIC.getKey()).setValueType(Metric.ValueType.INT.name())); + MetricDto bestValueMetricDto = db.measures() + .insertMetric(m -> m.setKey(METRIC_WITH_BEST_VALUE.getKey()).setValueType(Metric.ValueType.INT.name()).setOptimizedBestValue(true).setBestValue(0.0)); + metricRepository.add(stringMetricDto.getUuid(), STRING_METRIC); + metricRepository.add(intMetricDto.getUuid(), INT_METRIC); + metricRepository.add(bestValueMetricDto.getUuid(), METRIC_WITH_BEST_VALUE); + } + + @Test + void description() { + assertThat(step().getDescription()).isEqualTo("Persist measures"); + } + + @Test + void persist_measures_of_project_analysis() { + prepareProject(); + + // the computed measures + measureRepository.addRawMeasure(REF_1, STRING_METRIC.getKey(), newMeasureBuilder().create("project-value")); + measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("dir-value")); + measureRepository.addRawMeasure(REF_4, STRING_METRIC.getKey(), newMeasureBuilder().create("file-value")); + + step().execute(context); + + // all measures are persisted, from project to file + assertThat(db.countRowsOfTable("measures")).isEqualTo(3); + assertThat(selectMeasure("project-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "project-value")); + assertThat(selectMeasure("dir-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "dir-value")); + assertThat(selectMeasure("file-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "file-value")); + verifyInsertsOrUpdates(3); + } + + @Test + void persist_measures_of_portfolio_analysis() { + preparePortfolio(); + + // the computed measures + measureRepository.addRawMeasure(REF_1, STRING_METRIC.getKey(), newMeasureBuilder().create("view-value")); + measureRepository.addRawMeasure(REF_2, STRING_METRIC.getKey(), newMeasureBuilder().create("subview-value")); + measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("project-value")); + + step().execute(context); + + assertThat(db.countRowsOfTable("measures")).isEqualTo(3); + assertThat(selectMeasure("view-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "view-value")); + assertThat(selectMeasure("subview-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "subview-value")); + assertThat(selectMeasure("project-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).containsEntry(STRING_METRIC.getKey(), "project-value")); + verifyInsertsOrUpdates(3); + } + + @Test + void persists_large_number_of_measures() { + int num = 11; + List<ReportComponent> files = new LinkedList<>(); + String valuePrefix = "value".repeat(10); + + for (int i = 0; i < num; i++) { + files.add(ReportComponent.builder(FILE, i).setUuid("file-uuid" + i).build()); + } + Component project = ReportComponent.builder(Component.Type.PROJECT, -1).setUuid("project-uuid") + .addChildren(files.toArray(Component[]::new)) + .build(); + treeRootHolder.setRoot(project); + analysisMetadataHolder.setBaseAnalysis(new Analysis.Builder().setUuid("uuid").setCreatedAt(1L).build()); + insertMeasure("file-uuid0", "project-uuid", STRING_METRIC, valuePrefix + "0"); + + for (int i = 0; i < num; i++) { + measureRepository.addRawMeasure(i, STRING_METRIC.getKey(), newMeasureBuilder().create(valuePrefix + i)); + } + db.getSession().commit(); + + PersistMeasuresStep step = new PersistMeasuresStep(dbClient, metricRepository, treeRootHolder, measureRepository, + computeDuplicationDataMeasure, 100); + step.execute(context); + + // all measures are persisted, for project and all files + assertThat(db.countRowsOfTable("measures")).isEqualTo(num + 1); + verifyInsertsOrUpdates(num - 1); + verifyUnchanged(1); + verify(computeDuplicationDataMeasure, times(num)).compute(any(Component.class)); + } + + @Test + void do_not_persist_excluded_file_metrics() { + MetricDto fileComplexityMetric = + db.measures().insertMetric(m -> m.setKey(FILE_COMPLEXITY_DISTRIBUTION_KEY).setValueType(Metric.ValueType.STRING.name())); + MetricDto functionComplexityMetric = + db.measures().insertMetric(m -> m.setKey(FUNCTION_COMPLEXITY_DISTRIBUTION_KEY).setValueType(Metric.ValueType.INT.name())); + metricRepository.add(fileComplexityMetric.getUuid(), new Metric.Builder(FILE_COMPLEXITY_DISTRIBUTION_KEY, "File Distribution / " + + "Complexity", + Metric.ValueType.DISTRIB).create()); + metricRepository.add(functionComplexityMetric.getUuid(), new Metric.Builder(FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, "Function " + + "Distribution / Complexity", + Metric.ValueType.DISTRIB).create()); + + prepareProject(); + + // the computed measures + measureRepository.addRawMeasure(REF_1, FILE_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("project-value")); + measureRepository.addRawMeasure(REF_3, FILE_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("dir-value")); + measureRepository.addRawMeasure(REF_4, FILE_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("file-value")); + + measureRepository.addRawMeasure(REF_1, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("project-value")); + measureRepository.addRawMeasure(REF_3, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("dir-value")); + measureRepository.addRawMeasure(REF_4, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, newMeasureBuilder().create("file-value")); + + step().execute(context); + + // all measures are persisted, from project to file + assertThat(db.countRowsOfTable("measures")).isEqualTo(3); + + assertThat(selectMeasure("project-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).contains( + entry(FILE_COMPLEXITY_DISTRIBUTION_KEY, "project-value"), + entry(FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, "project-value"))); + assertThat(selectMeasure("dir-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()).contains( + entry(FILE_COMPLEXITY_DISTRIBUTION_KEY, "dir-value"), + entry(FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, "dir-value"))); + assertThat(selectMeasure("file-uuid")) + .hasValueSatisfying(measure -> assertThat(measure.getMetricValues()) + .doesNotContainKeys(FILE_COMPLEXITY_DISTRIBUTION_KEY, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY)); + + verifyInsertsOrUpdates(4); + } + + @Test + void measures_without_value_are_not_persisted() { + prepareProject(); + measureRepository.addRawMeasure(REF_1, STRING_METRIC.getKey(), newMeasureBuilder().createNoValue()); + measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().createNoValue()); + + step().execute(context); + + assertThat(selectMeasure("project-uuid")).hasValueSatisfying(measureDto -> assertThat(measureDto.getMetricValues()).isEmpty()); + verifyInsertsOrUpdates(0); + } + + @Test + void measures_on_leak_period_are_persisted() { + prepareProject(); + measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().create(42.0)); + + step().execute(context); + + assertThat(selectMeasure("project-uuid")) + .hasValueSatisfying(persistedMeasure -> assertThat(persistedMeasure.getMetricValues()).containsEntry(INT_METRIC.getKey(), 42.0)); + verifyInsertsOrUpdates(1); + } + + @Test + void do_not_persist_file_measures_with_best_value() { + prepareProject(); + // measure to be deleted because new value matches the metric best value + insertMeasure("file-uuid", "project-uuid", METRIC_WITH_BEST_VALUE, 123.0); + db.commit(); + + // project measure with metric best value -> persist with value 0 + measureRepository.addRawMeasure(REF_1, METRIC_WITH_BEST_VALUE.getKey(), newMeasureBuilder().create(0)); + // file measure with metric best value -> do not persist + measureRepository.addRawMeasure(REF_4, METRIC_WITH_BEST_VALUE.getKey(), newMeasureBuilder().create(0)); + + step().execute(context); + + assertThatMeasureDoesNotExist("file-uuid", METRIC_WITH_BEST_VALUE.getKey()); + + Optional<MeasureDto> persisted = dbClient.measureDao().selectMeasure(db.getSession(), "project-uuid"); + assertThat(persisted).isPresent(); + assertThat(persisted.get().getMetricValues()).containsEntry(METRIC_WITH_BEST_VALUE.getKey(), (double) 0); + + verifyInsertsOrUpdates(1); + } + + @Test + void do_not_persist_if_value_hash_unchanged() { + prepareProject(); + insertMeasure("file-uuid", "project-uuid", INT_METRIC, 123.0); + db.commit(); + + measureRepository.addRawMeasure(REF_4, INT_METRIC.getKey(), newMeasureBuilder().create(123L)); + + step().execute(context); + + verifyInsertsOrUpdates(0); + verifyUnchanged(1); + } + + @Test + void persist_if_value_hash_changed() { + prepareProject(); + insertMeasure("file-uuid", "project-uuid", INT_METRIC, 123.0); + db.commit(); + + measureRepository.addRawMeasure(REF_4, INT_METRIC.getKey(), newMeasureBuilder().create(124L)); + + step().execute(context); + + verifyInsertsOrUpdates(1); + verifyUnchanged(0); + } + + private void insertMeasure(String componentUuid, String projectUuid, Metric<?> metric, Object obj) { + MeasureDto measure = new MeasureDto() + .setComponentUuid(componentUuid) + .setBranchUuid(projectUuid) + .addValue(metric.getKey(), obj); + measure.computeJsonValueHash(); + dbClient.measureDao().insert(db.getSession(), measure); + } + + private void assertThatMeasureDoesNotExist(String componentUuid, String metricKey) { + assertThat(dbClient.measureDao().selectMeasure(db.getSession(), componentUuid)) + .hasValueSatisfying(measureDto -> assertThat(measureDto.getMetricValues()).doesNotContainKey(metricKey)); + } + + private void prepareProject() { + // tree of components as defined by scanner report + Component project = ReportComponent.builder(PROJECT, REF_1).setUuid("project-uuid") + .addChildren( + ReportComponent.builder(DIRECTORY, REF_3).setUuid("dir-uuid") + .addChildren( + ReportComponent.builder(FILE, REF_4).setUuid("file-uuid") + .build()) + .build()) + .build(); + treeRootHolder.setRoot(project); + analysisMetadataHolder.setProject(new Project(project.getUuid(), project.getKey(), project.getName(), project.getDescription(), + emptyList())); + + // components as persisted in db + insertComponent("project-key", "project-uuid"); + insertComponent("dir-key", "dir-uuid"); + insertComponent("file-key", "file-uuid"); + } + + private void preparePortfolio() { + // tree of components + Component portfolio = ViewsComponent.builder(VIEW, REF_1).setUuid("view-uuid") + .addChildren( + ViewsComponent.builder(SUBVIEW, REF_2).setUuid("subview-uuid") + .addChildren( + ViewsComponent.builder(PROJECT_VIEW, REF_3).setUuid("project-uuid") + .build()) + .build()) + .build(); + treeRootHolder.setRoot(portfolio); + + // components as persisted in db + ComponentDto portfolioDto = insertComponent("view-key", "view-uuid"); + insertComponent("subview-key", "subview-uuid"); + insertComponent("project-key", "project-uuid"); + analysisMetadataHolder.setProject(Project.from(portfolioDto)); + } + + private Optional<MeasureDto> selectMeasure(String componentUuid) { + return dbClient.measureDao().selectMeasure(db.getSession(), componentUuid); + } + + private ComponentDto insertComponent(String key, String uuid) { + ComponentDto componentDto = new ComponentDto() + .setKey(key) + .setUuid(uuid) + .setUuidPath(uuid + ".") + .setBranchUuid(uuid); + db.components().insertComponent(componentDto); + return componentDto; + } + + private PersistMeasuresStep step() { + return new PersistMeasuresStep(dbClient, metricRepository, treeRootHolder, measureRepository, computeDuplicationDataMeasure); + } + + private void verifyInsertsOrUpdates(int expectedInsertsOrUpdates) { + context.getStatistics().assertValue("insertsOrUpdates", expectedInsertsOrUpdates); + } + + private void verifyUnchanged(int expectedUnchanged) { + context.getStatistics().assertValue("unchanged", expectedUnchanged); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java index c7df68041a4..8f449fd604b 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java @@ -40,6 +40,7 @@ import org.sonar.ce.task.projectanalysis.component.ProjectPersister; import org.sonar.ce.task.projectanalysis.component.ReferenceBranchComponentUuids; import org.sonar.ce.task.projectanalysis.component.SiblingComponentsWithOpenIssues; import org.sonar.ce.task.projectanalysis.component.TreeRootHolderImpl; +import org.sonar.ce.task.projectanalysis.duplication.ComputeDuplicationDataMeasure; import org.sonar.ce.task.projectanalysis.duplication.CrossProjectDuplicationStatusHolderImpl; import org.sonar.ce.task.projectanalysis.duplication.DuplicationMeasures; import org.sonar.ce.task.projectanalysis.duplication.DuplicationRepositoryImpl; @@ -327,6 +328,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop // duplication IntegrateCrossProjectDuplications.class, DuplicationMeasures.class, + ComputeDuplicationDataMeasure.class, // views ViewIndex.class, diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasure.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasure.java new file mode 100644 index 00000000000..4dc7648a159 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasure.java @@ -0,0 +1,92 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.duplication; + +import java.util.Optional; +import org.apache.commons.text.StringEscapeUtils; +import org.sonar.ce.task.projectanalysis.component.Component; + +import static com.google.common.collect.Iterables.isEmpty; + +/** + * Compute duplication data measures on files, based on the {@link DuplicationRepository} + */ +public class ComputeDuplicationDataMeasure { + private final DuplicationRepository duplicationRepository; + + public ComputeDuplicationDataMeasure(DuplicationRepository duplicationRepository) { + this.duplicationRepository = duplicationRepository; + } + + public Optional<String> compute(Component file) { + Iterable<Duplication> duplications = duplicationRepository.getDuplications(file); + if (isEmpty(duplications)) { + return Optional.empty(); + } + return Optional.of(generateMeasure(file.getKey(), duplications)); + } + + private static String generateMeasure(String componentDbKey, Iterable<Duplication> duplications) { + StringBuilder xml = new StringBuilder(); + xml.append("<duplications>"); + for (Duplication duplication : duplications) { + xml.append("<g>"); + appendDuplication(xml, componentDbKey, duplication.getOriginal(), false); + for (Duplicate duplicate : duplication.getDuplicates()) { + processDuplicationBlock(xml, duplicate, componentDbKey); + } + xml.append("</g>"); + } + xml.append("</duplications>"); + return xml.toString(); + } + + private static void processDuplicationBlock(StringBuilder xml, Duplicate duplicate, String componentDbKey) { + if (duplicate instanceof InnerDuplicate) { + // Duplication is on the same file + appendDuplication(xml, componentDbKey, duplicate); + } else if (duplicate instanceof InExtendedProjectDuplicate inExtendedProjectDuplicate) { + // Duplication is on a different file that is not saved in the DB + appendDuplication(xml, inExtendedProjectDuplicate.getFile().getKey(), duplicate.getTextBlock(), true); + } else if (duplicate instanceof InProjectDuplicate inProjectDuplicate) { + // Duplication is on a different file + appendDuplication(xml, inProjectDuplicate.getFile().getKey(), duplicate); + } else if (duplicate instanceof CrossProjectDuplicate crossProjectDuplicate) { + // Only componentKey is set for cross project duplications + String crossProjectComponentKey = crossProjectDuplicate.getFileKey(); + appendDuplication(xml, crossProjectComponentKey, duplicate); + } else { + throw new IllegalArgumentException("Unsupported type of Duplicate " + duplicate.getClass().getName()); + } + } + + private static void appendDuplication(StringBuilder xml, String componentDbKey, Duplicate duplicate) { + appendDuplication(xml, componentDbKey, duplicate.getTextBlock(), false); + } + + private static void appendDuplication(StringBuilder xml, String componentDbKey, TextBlock textBlock, boolean disableLink) { + int length = textBlock.getEnd() - textBlock.getStart() + 1; + xml.append("<b s=\"").append(textBlock.getStart()) + .append("\" l=\"").append(length) + .append("\" t=\"").append(disableLink) + .append("\" r=\"").append(StringEscapeUtils.escapeXml10(componentDbKey)) + .append("\"/>"); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDto.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDto.java index fcd4b2d4a4b..c6714ff74f7 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDto.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDto.java @@ -20,12 +20,12 @@ package org.sonar.ce.task.projectanalysis.measure; import javax.annotation.CheckForNull; +import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolder; import org.sonar.ce.task.projectanalysis.metric.Metric; import org.sonar.db.measure.LiveMeasureDto; import org.sonar.db.measure.ProjectMeasureDto; -import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; -import org.sonar.ce.task.projectanalysis.component.TreeRootHolder; public class MeasureToMeasureDto { @@ -60,6 +60,14 @@ public class MeasureToMeasureDto { return out; } + public static Object getMeasureValue(Measure measure) { + Double doubleValue = valueAsDouble(measure); + if (doubleValue != null) { + return doubleValue; + } + return data(measure); + } + private static void setAlert(ProjectMeasureDto projectMeasureDto, QualityGateStatus qualityGateStatus) { projectMeasureDto.setAlertStatus(qualityGateStatus.getStatus().name()); projectMeasureDto.setAlertText(qualityGateStatus.getText()); diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistMeasuresStep.java new file mode 100644 index 00000000000..456d72e1286 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistMeasuresStep.java @@ -0,0 +1,208 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit; +import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolder; +import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter; +import org.sonar.ce.task.projectanalysis.duplication.ComputeDuplicationDataMeasure; +import org.sonar.ce.task.projectanalysis.measure.BestValueOptimization; +import org.sonar.ce.task.projectanalysis.measure.Measure; +import org.sonar.ce.task.projectanalysis.measure.MeasureRepository; +import org.sonar.ce.task.projectanalysis.measure.MeasureToMeasureDto; +import org.sonar.ce.task.projectanalysis.metric.Metric; +import org.sonar.ce.task.projectanalysis.metric.MetricRepository; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.measure.MeasureDto; +import org.sonar.db.measure.MeasureHash; +import org.springframework.beans.factory.annotation.Autowired; + +import static org.sonar.api.measures.CoreMetrics.DUPLICATIONS_DATA_KEY; +import static org.sonar.api.measures.CoreMetrics.FILE_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER; + +public class PersistMeasuresStep implements ComputationStep { + + /** + * List of metrics that should not be persisted on file measure. + */ + private static final Set<String> NOT_TO_PERSIST_ON_FILE_METRIC_KEYS = Set.of( + FILE_COMPLEXITY_DISTRIBUTION_KEY, + FUNCTION_COMPLEXITY_DISTRIBUTION_KEY); + + // 50 mb + private static final int MAX_TRANSACTION_SIZE = 50_000_000; + private static final Predicate<Measure> NON_EMPTY_MEASURE = measure -> + measure.getValueType() != Measure.ValueType.NO_VALUE || measure.getData() != null; + + private final DbClient dbClient; + private final MetricRepository metricRepository; + private final TreeRootHolder treeRootHolder; + private final MeasureRepository measureRepository; + private final ComputeDuplicationDataMeasure computeDuplicationDataMeasure; + private final int maxTransactionSize; + + @Autowired + public PersistMeasuresStep(DbClient dbClient, MetricRepository metricRepository, TreeRootHolder treeRootHolder, + MeasureRepository measureRepository, @Nullable ComputeDuplicationDataMeasure computeDuplicationDataMeasure) { + this(dbClient, metricRepository, treeRootHolder, measureRepository, computeDuplicationDataMeasure, MAX_TRANSACTION_SIZE); + } + + PersistMeasuresStep(DbClient dbClient, MetricRepository metricRepository, TreeRootHolder treeRootHolder, + MeasureRepository measureRepository, @Nullable ComputeDuplicationDataMeasure computeDuplicationDataMeasure, int maxTransactionSize) { + this.dbClient = dbClient; + this.metricRepository = metricRepository; + this.treeRootHolder = treeRootHolder; + this.measureRepository = measureRepository; + this.computeDuplicationDataMeasure = computeDuplicationDataMeasure; + this.maxTransactionSize = maxTransactionSize; + } + + @Override + public String getDescription() { + return "Persist measures"; + } + + @Override + public void execute(ComputationStep.Context context) { + Component root = treeRootHolder.getRoot(); + CollectComponentsVisitor visitor = new CollectComponentsVisitor(); + new DepthTraversalTypeAwareCrawler(visitor).visit(root); + + Set<MeasureHash> dbMeasureHashes = getDBMeasureHashes(); + Set<String> dbComponents = dbMeasureHashes.stream().map(MeasureHash::componentUuid).collect(Collectors.toSet()); + + List<MeasureDto> inserts = new LinkedList<>(); + List<MeasureDto> updates = new LinkedList<>(); + int insertsOrUpdates = 0; + int unchanged = 0; + int size = 0; + + for (Component component : visitor.components) { + MeasureDto measure = createMeasure(component); + + if (dbMeasureHashes.contains(new MeasureHash(measure.getComponentUuid(), measure.computeJsonValueHash()))) { + unchanged += measure.getMetricValues().size(); + } else { + if (dbComponents.contains(measure.getComponentUuid())) { + updates.add(measure); + } else { + inserts.add(measure); + } + size += measure.getJsonValue().length(); + insertsOrUpdates += measure.getMetricValues().size(); + } + + if (size > maxTransactionSize) { + persist(inserts, updates); + inserts.clear(); + updates.clear(); + size = 0; + } + } + persist(inserts, updates); + + context.getStatistics() + .add("insertsOrUpdates", insertsOrUpdates) + .add("unchanged", unchanged); + } + + private Set<MeasureHash> getDBMeasureHashes() { + try (DbSession dbSession = dbClient.openSession(false)) { + return dbClient.measureDao().selectBranchMeasureHashes(dbSession, treeRootHolder.getRoot().getUuid()); + } + } + + private MeasureDto createMeasure(Component component) { + MeasureDto measureDto = new MeasureDto(); + measureDto.setComponentUuid(component.getUuid()); + measureDto.setBranchUuid(treeRootHolder.getRoot().getUuid()); + + Map<String, Measure> measures = measureRepository.getRawMeasures(component); + for (Map.Entry<String, Measure> measuresByMetricKey : measures.entrySet()) { + String metricKey = measuresByMetricKey.getKey(); + if (NOT_TO_PERSIST_ON_FILE_METRIC_KEYS.contains(metricKey) && component.getType() == Component.Type.FILE) { + continue; + } + Metric metric = metricRepository.getByKey(metricKey); + Predicate<Measure> notBestValueOptimized = BestValueOptimization.from(metric, component).negate(); + Measure measure = measuresByMetricKey.getValue(); + Stream.of(measure) + .filter(NON_EMPTY_MEASURE) + .filter(notBestValueOptimized) + .map(MeasureToMeasureDto::getMeasureValue) + .filter(Objects::nonNull) + .forEach(value -> measureDto.addValue(metric.getKey(), value)); + + if (component.getType() == Component.Type.FILE) { + if (computeDuplicationDataMeasure == null) { + throw new IllegalStateException("ComputeDuplicationDataMeasure not initialized in container"); + } + computeDuplicationDataMeasure.compute(component) + .ifPresent(duplicationData -> measureDto.addValue(DUPLICATIONS_DATA_KEY, duplicationData)); + } + } + + return measureDto; + } + + private void persist(Collection<MeasureDto> inserts, Collection<MeasureDto> updates) { + if (inserts.isEmpty() && updates.isEmpty()) { + return; + } + try (DbSession dbSession = dbClient.openSession(true)) { + for (MeasureDto m : inserts) { + dbClient.measureDao().insert(dbSession, m); + } + for (MeasureDto m : updates) { + dbClient.measureDao().update(dbSession, m); + } + dbSession.commit(); + } + } + + private static class CollectComponentsVisitor extends TypeAwareVisitorAdapter { + private final List<Component> components = new LinkedList<>(); + + private CollectComponentsVisitor() { + super(CrawlerDepthLimit.LEAVES, PRE_ORDER); + } + + @Override + public void visitAny(Component component) { + components.add(component); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java index 34df5531d9d..a5cf39e5c60 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java @@ -107,6 +107,7 @@ public class ReportComputationSteps extends AbstractComputationSteps { PersistAnalysisPropertiesStep.class, PersistProjectMeasuresStep.class, PersistLiveMeasuresStep.class, + PersistMeasuresStep.class, PersistDuplicationDataStep.class, PersistAdHocRulesStep.class, PersistCveStep.class, diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasureTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasureTest.java new file mode 100644 index 00000000000..5e10730ae16 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/duplication/ComputeDuplicationDataMeasureTest.java @@ -0,0 +1,112 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.duplication; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; +import org.sonar.db.DbTester; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.FILE; +import static org.sonar.ce.task.projectanalysis.component.Component.Type.PROJECT; +import static org.sonar.ce.task.projectanalysis.component.ReportComponent.builder; + +class ComputeDuplicationDataMeasureTest { + + private static final int ROOT_REF = 1; + private static final String PROJECT_KEY = "PROJECT_KEY"; + private static final String PROJECT_UUID = "u1"; + + private static final int FILE_1_REF = 2; + private static final String FILE_1_KEY = "FILE_1_KEY"; + private static final String FILE_1_UUID = "u2"; + + private static final int FILE_2_REF = 3; + private static final String FILE_2_KEY = "FILE_2_KEY"; + private static final String FILE_2_UUID = "u3"; + + @RegisterExtension + public DbTester db = DbTester.create(System2.INSTANCE); + + private final Component file1 = builder(FILE, FILE_1_REF).setKey(FILE_1_KEY).setUuid(FILE_1_UUID).build(); + private final Component file2 = builder(FILE, FILE_2_REF).setKey(FILE_2_KEY).setUuid(FILE_2_UUID).build(); + + @RegisterExtension + public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule() + .setRoot( + builder(PROJECT, ROOT_REF).setKey(PROJECT_KEY).setUuid(PROJECT_UUID) + .addChildren(file1, file2) + .build()); + + @RegisterExtension + public DuplicationRepositoryRule duplicationRepository = DuplicationRepositoryRule.create(treeRootHolder); + + @Test + void nothing_to_persist_when_no_duplication() { + assertThat(underTest().compute(file1)).isEmpty(); + } + + @Test + void compute_duplications_on_same_file() { + duplicationRepository.addDuplication(FILE_1_REF, new TextBlock(1, 5), new TextBlock(6, 10)); + + assertThat(underTest().compute(file1)) + .contains("<duplications><g><b s=\"1\" l=\"5\" t=\"false\" r=\"" + FILE_1_KEY + "\"/><b s=\"6\" l=\"5\" t=\"false\" r=\"" + + FILE_1_KEY + "\"/></g></duplications>"); + } + + @Test + void compute_duplications_on_different_files() { + duplicationRepository.addDuplication(FILE_1_REF, new TextBlock(1, 5), FILE_2_REF, new TextBlock(6, 10)); + + assertThat(underTest().compute(file1)) + .contains("<duplications><g><b s=\"1\" l=\"5\" t=\"false\" r=\"" + FILE_1_KEY + "\"/><b s=\"6\" l=\"5\" t=\"false\" r=\"" + + FILE_2_KEY + "\"/></g></duplications>"); + assertThat(underTest().compute(file2)).isEmpty(); + } + + @Test + void compute_duplications_on_unchanged_file() { + duplicationRepository.addExtendedProjectDuplication(FILE_1_REF, new TextBlock(1, 5), FILE_2_REF, new TextBlock(6, 10)); + + assertThat(underTest().compute(file1)) + .contains("<duplications><g><b s=\"1\" l=\"5\" t=\"false\" r=\"" + FILE_1_KEY + "\"/><b s=\"6\" l=\"5\" t=\"true\" r=\"" + + FILE_2_KEY + "\"/></g></duplications>"); + assertThat(underTest().compute(file2)).isEmpty(); + } + + @Test + void compute_duplications_on_different_projects() { + String fileKeyFromOtherProject = "PROJECT2_KEY:file2"; + duplicationRepository.addCrossProjectDuplication(FILE_1_REF, new TextBlock(1, 5), fileKeyFromOtherProject, new TextBlock(6, 10)); + + assertThat(underTest().compute(file1)) + .contains("<duplications><g><b s=\"1\" l=\"5\" t=\"false\" r=\"" + FILE_1_KEY + "\"/><b s=\"6\" l=\"5\" t=\"false\" r=\"" + + fileKeyFromOtherProject + "\"/></g></duplications>"); + assertThat(underTest().compute(file2)).isEmpty(); + } + + private ComputeDuplicationDataMeasure underTest() { + return new ComputeDuplicationDataMeasure(duplicationRepository); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDtoTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDtoTest.java index 763e5ddb544..2314fc36f53 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDtoTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/measure/MeasureToMeasureDtoTest.java @@ -37,6 +37,7 @@ import org.sonar.db.measure.ProjectMeasureDto; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.sonar.ce.task.projectanalysis.measure.MeasureToMeasureDto.getMeasureValue; @RunWith(DataProviderRunner.class) public class MeasureToMeasureDtoTest { @@ -181,4 +182,42 @@ public class MeasureToMeasureDtoTest { assertThat(liveMeasureDto.getTextValue()).isEqualTo(Measure.Level.OK.name()); } + + @Test + public void getMeasureValue_returns_null_if_measure_is_empty() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().createNoValue())).isNull(); + } + + @Test + public void getMeasureValue_maps_value_to_1_or_0_and_data_from_data_field_for_BOOLEAN_metric() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().create(true, SOME_DATA))).isEqualTo(1d); + assertThat(getMeasureValue(Measure.newMeasureBuilder().create(false, SOME_DATA))).isEqualTo(0d); + } + + @Test + public void getMeasureValue_maps_value_and_data_from_data_field_for_INT_metric() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().create(123, SOME_DATA))).isEqualTo(123.0); + } + + @Test + public void getMeasureValue_maps_value_and_data_from_data_field_for_LONG_metric() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().create((long) 456, SOME_DATA))).isEqualTo(456.0); + } + + @Test + public void getMeasureValue_maps_value_and_data_from_data_field_for_DOUBLE_metric() { + assertThat(getMeasureValue(Measure.newMeasureBuilder().create(789, 1, SOME_DATA))).isEqualTo(789.0); + } + + @Test + public void getMeasureValue_maps_to_only_data_for_STRING_metric() { + assertThat(getMeasureValue( + Measure.newMeasureBuilder().create(SOME_STRING))).isEqualTo(SOME_STRING); + } + + @Test + public void getMeasureValue_maps_name_of_Level_to_data_and_has_no_value_for_LEVEL_metric() { + assertThat(getMeasureValue( + Measure.newMeasureBuilder().create(Measure.Level.OK))).isEqualTo(Measure.Level.OK.name()); + } } diff --git a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java index d49418ecf62..e72b6326b16 100644 --- a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java +++ b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java @@ -67,6 +67,7 @@ public final class SqTables { "issues_impacts", "issue_changes", "live_measures", + "measures", "metrics", "new_code_periods", "new_code_reference_issues", diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java index f11922bff2a..c0852b111a5 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java @@ -51,6 +51,7 @@ import org.sonar.db.issue.IssueChangeDao; import org.sonar.db.issue.IssueDao; import org.sonar.db.issue.IssueFixedDao; import org.sonar.db.measure.LiveMeasureDao; +import org.sonar.db.measure.MeasureDao; import org.sonar.db.measure.ProjectMeasureDao; import org.sonar.db.metric.MetricDao; import org.sonar.db.newcodeperiod.NewCodePeriodDao; @@ -153,6 +154,7 @@ public class DaoModule extends Module { IssueDao.class, IssueFixedDao.class, IssuesDependencyDao.class, + MeasureDao.class, LiveMeasureDao.class, ProjectMeasureDao.class, MetricDao.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java index d04aff1df52..200f3547cb4 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java @@ -51,6 +51,7 @@ import org.sonar.db.issue.IssueChangeDao; import org.sonar.db.issue.IssueDao; import org.sonar.db.issue.IssueFixedDao; import org.sonar.db.measure.LiveMeasureDao; +import org.sonar.db.measure.MeasureDao; import org.sonar.db.measure.ProjectMeasureDao; import org.sonar.db.metric.MetricDao; import org.sonar.db.newcodeperiod.NewCodePeriodDao; @@ -129,6 +130,7 @@ public class DbClient { private final SnapshotDao snapshotDao; private final ComponentDao componentDao; private final ComponentKeyUpdaterDao componentKeyUpdaterDao; + private final MeasureDao measureDao; private final ProjectMeasureDao projectMeasureDao; private final UserDao userDao; private final UserGroupDao userGroupDao; @@ -225,6 +227,7 @@ public class DbClient { snapshotDao = getDao(map, SnapshotDao.class); componentDao = getDao(map, ComponentDao.class); componentKeyUpdaterDao = getDao(map, ComponentKeyUpdaterDao.class); + measureDao = getDao(map, MeasureDao.class); projectMeasureDao = getDao(map, ProjectMeasureDao.class); userDao = getDao(map, UserDao.class); userGroupDao = getDao(map, UserGroupDao.class); @@ -397,6 +400,10 @@ public class DbClient { return componentKeyUpdaterDao; } + public MeasureDao measureDao() { + return measureDao; + } + public ProjectMeasureDao projectMeasureDao() { return projectMeasureDao; } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java index 4191942d8d0..9aafec712ab 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java @@ -89,6 +89,7 @@ import org.sonar.db.issue.NewCodeReferenceIssueDto; import org.sonar.db.issue.PrIssueDto; import org.sonar.db.measure.LargestBranchNclocDto; import org.sonar.db.measure.LiveMeasureMapper; +import org.sonar.db.measure.MeasureMapper; import org.sonar.db.measure.ProjectLocDistributionDto; import org.sonar.db.measure.ProjectMeasureDto; import org.sonar.db.measure.ProjectMeasureMapper; @@ -316,6 +317,7 @@ public class MyBatis { IssueMapper.class, IssueFixedMapper.class, IssuesDependencyMapper.class, + MeasureMapper.class, ProjectMeasureMapper.class, MetricMapper.class, NewCodePeriodMapper.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java new file mode 100644 index 00000000000..eebceb810ad --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java @@ -0,0 +1,61 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.sonar.api.utils.System2; +import org.sonar.db.Dao; +import org.sonar.db.DbSession; + +public class MeasureDao implements Dao { + + private final System2 system2; + + public MeasureDao(System2 system2) { + this.system2 = system2; + } + + public int insert(DbSession dbSession, MeasureDto dto) { + return mapper(dbSession).insert(dto, system2.now()); + } + + public int update(DbSession dbSession, MeasureDto dto) { + return mapper(dbSession).update(dto, system2.now()); + } + + public Optional<MeasureDto> selectMeasure(DbSession dbSession, String componentUuid) { + List<MeasureDto> measures = mapper(dbSession).selectByComponentUuids(List.of(componentUuid)); + if (!measures.isEmpty()) { + // component_uuid column is unique. List can't have more than 1 item. + return Optional.of(measures.get(0)); + } + return Optional.empty(); + } + + public Set<MeasureHash> selectBranchMeasureHashes(DbSession dbSession, String branchUuid) { + return mapper(dbSession).selectBranchMeasureHashes(branchUuid); + } + + private static MeasureMapper mapper(DbSession dbSession) { + return dbSession.getMapper(MeasureMapper.class); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDto.java new file mode 100644 index 00000000000..8e93bd9a9a5 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDto.java @@ -0,0 +1,101 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.util.Map; +import java.util.TreeMap; +import org.apache.commons.codec.digest.MurmurHash3; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class MeasureDto { + + private static final Gson GSON = new Gson(); + + private String componentUuid; + private String branchUuid; + // measures are kept sorted by metric key so that the value hash is consistent + private Map<String, Object> metricValues = new TreeMap<>(); + private Long jsonValueHash; + + public MeasureDto() { + // empty constructor + } + + public String getComponentUuid() { + return componentUuid; + } + + public MeasureDto setComponentUuid(String s) { + this.componentUuid = s; + return this; + } + + public String getBranchUuid() { + return branchUuid; + } + + public MeasureDto setBranchUuid(String s) { + this.branchUuid = s; + return this; + } + + public Map<String, Object> getMetricValues() { + return metricValues; + } + + public MeasureDto addValue(String metricKey, Object value) { + metricValues.put(metricKey, value); + return this; + } + + // used by MyBatis mapper + public String getJsonValue() { + return GSON.toJson(metricValues); + } + + // used by MyBatis mapper + public MeasureDto setJsonValue(String jsonValue) { + metricValues = GSON.fromJson(jsonValue, new TypeToken<TreeMap<String, Object>>() { + }.getType()); + return this; + } + + public Long getJsonValueHash() { + return jsonValueHash; + } + + public long computeJsonValueHash() { + jsonValueHash = MurmurHash3.hash128(getJsonValue().getBytes(UTF_8))[0]; + return jsonValueHash; + } + + @Override + public String toString() { + return "MeasureDto{" + + "componentUuid='" + componentUuid + '\'' + + ", branchUuid='" + branchUuid + '\'' + + ", metricValues=" + metricValues + + ", jsonValueHash=" + jsonValueHash + + '}'; + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureHash.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureHash.java new file mode 100644 index 00000000000..65b73a665ba --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureHash.java @@ -0,0 +1,32 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.Comparator; + +public record MeasureHash(String componentUuid, Long jsonValueHash) implements Comparable<MeasureHash> { + + @Override + public int compareTo(MeasureHash o) { + return Comparator.comparing(MeasureHash::componentUuid) + .thenComparing(MeasureHash::jsonValueHash) + .compare(this, o); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java new file mode 100644 index 00000000000..69b4c2fefbe --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java @@ -0,0 +1,40 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.Collection; +import java.util.List; +import java.util.Set; +import org.apache.ibatis.annotations.Param; + +public interface MeasureMapper { + + int insert( + @Param("dto") MeasureDto dto, + @Param("now") long now); + + int update( + @Param("dto") MeasureDto dto, + @Param("now") long now); + + List<MeasureDto> selectByComponentUuids(@Param("componentUuids") Collection<String> componentUuids); + + Set<MeasureHash> selectBranchMeasureHashes(@Param("branchUuid") String branchUuid); +} diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml new file mode 100644 index 00000000000..8fc634a5f1c --- /dev/null +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "mybatis-3-mapper.dtd"> + +<mapper namespace="org.sonar.db.measure.MeasureMapper"> + + <insert id="insert" parameterType="map" useGeneratedKeys="false"> + insert into measures ( + component_uuid, + branch_uuid, + json_value, + json_value_hash, + created_at, + updated_at + ) values ( + #{dto.componentUuid, jdbcType=VARCHAR}, + #{dto.branchUuid, jdbcType=VARCHAR}, + #{dto.jsonValue, jdbcType=VARCHAR}, + #{dto.jsonValueHash, jdbcType=BIGINT}, + #{now, jdbcType=BIGINT}, + #{now, jdbcType=BIGINT} + ) + </insert> + + <update id="update" parameterType="map" useGeneratedKeys="false"> + update measures set + json_value = #{dto.jsonValue, jdbcType=VARCHAR}, + json_value_hash = #{dto.jsonValueHash, jdbcType=BIGINT}, + updated_at = #{now, jdbcType=BIGINT} + where component_uuid = #{dto.componentUuid, jdbcType=VARCHAR} + </update> + + <sql id="columns"> + m.component_uuid as componentUuid, + m.branch_uuid as branchUuid, + m.json_value as jsonValue, + m.json_value_hash as jsonValueHash + </sql> + + <select id="selectByComponentUuids" parameterType="map" resultType="org.sonar.db.measure.MeasureDto"> + select + <include refid="columns"/> + from measures m + where + m.component_uuid in + <foreach item="componentUuid" collection="componentUuids" open="(" separator="," close=")"> + #{componentUuid, jdbcType=VARCHAR} + </foreach> + </select> + + <select id="selectBranchMeasureHashes" resultType="org.sonar.db.measure.MeasureHash"> + select component_uuid, json_value_hash + from measures + where branch_uuid = #{branchUuid, jdbcType=VARCHAR} + </select> + +</mapper> diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureDaoTest.java new file mode 100644 index 00000000000..1fdd6d7b9c1 --- /dev/null +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureDaoTest.java @@ -0,0 +1,107 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.sonar.api.utils.System2; +import org.sonar.db.DbTester; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.db.measure.MeasureTesting.newMeasure; + +class MeasureDaoTest { + + @RegisterExtension + public final DbTester db = DbTester.create(System2.INSTANCE); + + private final MeasureDao underTest = db.getDbClient().measureDao(); + + @Test + void insert_measure() { + MeasureDto dto = newMeasure(); + int count = underTest.insert(db.getSession(), dto); + assertThat(count).isEqualTo(1); + verifyTableSize(1); + verifyPersisted(dto); + } + + @Test + void update_measure() { + MeasureDto dto = newMeasure(); + underTest.insert(db.getSession(), dto); + + dto.addValue("metric1", "value1"); + dto.computeJsonValueHash(); + int count = underTest.update(db.getSession(), dto); + + assertThat(count).isEqualTo(1); + verifyTableSize(1); + verifyPersisted(dto); + } + + @Test + void select_measure() { + MeasureDto measure1 = newMeasure(); + MeasureDto measure2 = newMeasure(); + underTest.insert(db.getSession(), measure1); + underTest.insert(db.getSession(), measure2); + + assertThat(underTest.selectMeasure(db.getSession(), measure1.getComponentUuid())) + .hasValueSatisfying(selected -> assertThat(selected).usingRecursiveComparison().isEqualTo(measure1)); + assertThat(underTest.selectMeasure(db.getSession(), "unknown-component")).isEmpty(); + } + + @Test + void select_branch_measure_hashes() { + MeasureDto measure1 = new MeasureDto() + .setComponentUuid("c1") + .setBranchUuid("b1") + .addValue("metric1", "value1"); + MeasureDto measure2 = new MeasureDto() + .setComponentUuid("c2") + .setBranchUuid("b1") + .addValue("metric2", "value2"); + MeasureDto measure3 = new MeasureDto() + .setComponentUuid("c3") + .setBranchUuid("b3") + .addValue("metric3", "value3"); + long hash1 = measure1.computeJsonValueHash(); + long hash2 = measure2.computeJsonValueHash(); + measure3.computeJsonValueHash(); + + underTest.insert(db.getSession(), measure1); + underTest.insert(db.getSession(), measure2); + underTest.insert(db.getSession(), measure3); + + assertThat(underTest.selectBranchMeasureHashes(db.getSession(), "b1")) + .containsOnly(new MeasureHash("c1", hash1), new MeasureHash("c2", hash2)); + } + + private void verifyTableSize(int expectedSize) { + assertThat(db.countRowsOfTable(db.getSession(), "measures")).isEqualTo(expectedSize); + } + + private void verifyPersisted(MeasureDto dto) { + assertThat(underTest.selectMeasure(db.getSession(), dto.getComponentUuid())).hasValueSatisfying(selected -> { + assertThat(selected).usingRecursiveComparison().isEqualTo(dto); + }); + } +} diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureDtoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureDtoTest.java new file mode 100644 index 00000000000..9299402942d --- /dev/null +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureDtoTest.java @@ -0,0 +1,68 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.Map; +import java.util.TreeMap; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class MeasureDtoTest { + + @Test + void compute_json_value_hash() { + MeasureDto measureDto = new MeasureDto(); + measureDto.setJsonValue("{\"key\":\"value\"}"); + assertThat(measureDto.getJsonValueHash()).isNull(); + assertThat(measureDto.computeJsonValueHash()).isEqualTo(2887272982314571750L); + assertThat(measureDto.getJsonValueHash()).isEqualTo(2887272982314571750L); + } + + @Test + void getMetricValues_returns_all_values_ordered() { + MeasureDto measureDto = new MeasureDto() + .addValue("string-metric", "value") + .addValue("int-metric", 1) + .addValue("long-metric", 2L); + assertThat(measureDto.getMetricValues()).containsExactlyEntriesOf(new TreeMap<>(Map.of( + "int-metric", 1, + "long-metric", 2L, + "string-metric", "value" + ))); + } + + @Test + void toString_prints_all_fields() { + MeasureDto measureDto = new MeasureDto() + .setBranchUuid("branch-uuid") + .setComponentUuid("component-uuid") + .addValue("int-metric", 12) + .addValue("double-metric", 34.5) + .addValue("boolean-metric", true) + .addValue("string-metric", "value"); + measureDto.computeJsonValueHash(); + + assertThat(measureDto).hasToString("MeasureDto{componentUuid='component-uuid'" + + ", branchUuid='branch-uuid'" + + ", metricValues={boolean-metric=true, double-metric=34.5, int-metric=12, string-metric=value}" + + ", jsonValueHash=-1071134275520515337}"); + } +} diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureHashTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureHashTest.java new file mode 100644 index 00000000000..ff11db17994 --- /dev/null +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/MeasureHashTest.java @@ -0,0 +1,54 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +import java.util.List; +import java.util.TreeSet; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class MeasureHashTest { + + @Test + void hashCode_depends_on_both_fields() { + MeasureHash measureHash1 = new MeasureHash("component1", 123L); + MeasureHash measureHash2 = new MeasureHash("component", 123L); + MeasureHash measureHash3 = new MeasureHash("component", 124L); + + assertThat(measureHash1) + .doesNotHaveSameHashCodeAs(measureHash2) + .doesNotHaveSameHashCodeAs(measureHash3) + .isNotEqualTo(measureHash2) + .isNotEqualTo(measureHash3); + } + + @Test + void sort_by_component_and_hash() { + MeasureHash measureHash1 = new MeasureHash("A", 1L); + MeasureHash measureHash2 = new MeasureHash("A", 2L); + MeasureHash measureHash3 = new MeasureHash("B", 1L); + MeasureHash measureHash4 = new MeasureHash("B", 2L); + + TreeSet<MeasureHash> set = new TreeSet<>(List.of(measureHash1, measureHash2, measureHash3, measureHash4)); + + assertThat(set).containsExactly(measureHash1, measureHash2, measureHash3, measureHash4); + } +} diff --git a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureTesting.java b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureTesting.java index a01b6d5ba26..89db3b7adc5 100644 --- a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureTesting.java +++ b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/measure/MeasureTesting.java @@ -90,4 +90,22 @@ public class MeasureTesting { .setData(String.valueOf(cursor++)) .setValue((double) cursor++); } + + public static MeasureDto newMeasure() { + MeasureDto measureDto = new MeasureDto() + .setComponentUuid(String.valueOf(cursor++)) + .setBranchUuid(String.valueOf(cursor++)) + .addValue("metric" + cursor++, (double) cursor++); + measureDto.computeJsonValueHash(); + return measureDto; + } + + public static MeasureDto newMeasure(ComponentDto component, MetricDto metric, Object value) { + MeasureDto measureDto = new MeasureDto() + .setComponentUuid(component.uuid()) + .setBranchUuid(component.branchUuid()) + .addValue(metric.getKey(), value); + measureDto.computeJsonValueHash(); + return measureDto; + } } |