aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDuarte Meneses <duarte.meneses@sonarsource.com>2022-06-22 16:17:26 -0500
committersonartech <sonartech@sonarsource.com>2022-07-23 20:02:53 +0000
commit0fb5e45d935ad212aa3fe32202c33dd8395078c5 (patch)
tree4c2afc044dc034f9c70a11b58f02a762b3ba3448
parent1220331cf405fb916a005284120b0ed02ea67ac2 (diff)
downloadsonarqube-0fb5e45d935ad212aa3fe32202c33dd8395078c5.tar.gz
sonarqube-0fb5e45d935ad212aa3fe32202c33dd8395078c5.zip
SONAR-17044 Optimize Compute Engine issue tracking and persisting of measures when file is marked as unchanged
-rw-r--r--build.gradle2
-rw-r--r--plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java3
-rw-r--r--plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/MarkAsUnchangedSensor.java50
-rw-r--r--plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/OneIssuePerLineSensor.java1
-rw-r--r--plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/MarkAsUnchangedSensorTest.java74
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java3
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java11
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java30
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java98
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepository.java27
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImpl.java48
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java4
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitor.java38
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImpl.java13
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStep.java23
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStep.java68
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java106
-rw-r--r--server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java1
-rw-r--r--server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilderTest.java25
-rw-r--r--server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileAttributesTest.java20
-rw-r--r--server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java184
-rw-r--r--server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImplTest.java62
-rw-r--r--server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitorTest.java48
-rw-r--r--server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java46
-rw-r--r--server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStepTest.java55
-rw-r--r--server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStepTest.java126
-rw-r--r--server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java44
-rw-r--r--server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/component/ReportComponent.java2
-rw-r--r--server/sonar-db-dao/src/main/java/org/sonar/db/source/FileHashesDto.java92
-rw-r--r--server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java7
-rw-r--r--server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDto.java41
-rw-r--r--server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java3
-rw-r--r--server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml5
-rw-r--r--server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java39
-rw-r--r--sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputComponent.java2
-rw-r--r--sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java12
-rw-r--r--sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java7
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java1
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java4
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java3
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java11
-rw-r--r--sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java10
-rw-r--r--sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java10
-rw-r--r--sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java5
-rw-r--r--sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java5
-rw-r--r--sonar-scanner-protocol/src/main/protobuf/scanner_report.proto1
46 files changed, 1296 insertions, 174 deletions
diff --git a/build.gradle b/build.gradle
index 9339b64e526..32fa4bc083e 100644
--- a/build.gradle
+++ b/build.gradle
@@ -178,7 +178,7 @@ subprojects {
dependency 'org.sonarsource.kotlin:sonar-kotlin-plugin:2.9.0.1147'
dependency 'org.sonarsource.slang:sonar-ruby-plugin:1.9.0.3429'
dependency 'org.sonarsource.slang:sonar-scala-plugin:1.9.0.3429'
- dependency 'org.sonarsource.api.plugin:sonar-plugin-api:9.8.0.203'
+ dependency 'org.sonarsource.api.plugin:sonar-plugin-api:9.9.0.228'
dependency 'org.sonarsource.xml:sonar-xml-plugin:2.5.0.3376'
dependency 'org.sonarsource.iac:sonar-iac-plugin:1.7.0.2012'
dependency 'org.sonarsource.text:sonar-text-plugin:1.1.0.282'
diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
index 9f000f49013..ef3d0cce1e5 100644
--- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
@@ -46,6 +46,8 @@ import org.sonar.xoo.rule.HasTagSensor;
import org.sonar.xoo.rule.hotspot.HotspotWithSingleContextSensor;
import org.sonar.xoo.rule.hotspot.HotspotWithoutContextSensor;
import org.sonar.xoo.rule.hotspot.HotspotWithContextsSensor;
+import org.sonar.xoo.rule.HotspotSensor;
+import org.sonar.xoo.rule.MarkAsUnchangedSensor;
import org.sonar.xoo.rule.MultilineIssuesSensor;
import org.sonar.xoo.rule.NoSonarSensor;
import org.sonar.xoo.rule.OneBlockerIssuePerFileSensor;
@@ -183,6 +185,7 @@ public class XooPlugin implements Plugin {
AnalysisErrorSensor.class,
// Other
+ MarkAsUnchangedSensor.class,
XooProjectBuilder.class,
XooPostJob.class,
XooIssueFilter.class,
diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/MarkAsUnchangedSensor.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/MarkAsUnchangedSensor.java
new file mode 100644
index 00000000000..68ece9802a8
--- /dev/null
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/MarkAsUnchangedSensor.java
@@ -0,0 +1,50 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.xoo.rule;
+
+import org.sonar.api.batch.fs.FilePredicates;
+import org.sonar.api.batch.fs.FileSystem;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.sensor.Sensor;
+import org.sonar.api.batch.sensor.SensorContext;
+import org.sonar.api.batch.sensor.SensorDescriptor;
+import org.sonar.xoo.Xoo;
+import org.sonar.xoo.Xoo2;
+
+public class MarkAsUnchangedSensor implements Sensor {
+ public static final String ACTIVATE_MARK_AS_UNCHANGED = "sonar.markAsUnchanged";
+
+ @Override
+ public void describe(SensorDescriptor descriptor) {
+ descriptor.name("Mark As Unchanged Sensor")
+ .onlyOnLanguages(Xoo.KEY, Xoo2.KEY)
+ .onlyWhenConfiguration(c -> c.getBoolean(ACTIVATE_MARK_AS_UNCHANGED).orElse(false))
+ .processesFilesIndependently();
+ }
+
+ @Override
+ public void execute(SensorContext context) {
+ FileSystem fs = context.fileSystem();
+ FilePredicates p = fs.predicates();
+ for (InputFile f : fs.inputFiles(p.all())) {
+ context.markAsUnchanged(f);
+ }
+ }
+}
diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/OneIssuePerLineSensor.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/OneIssuePerLineSensor.java
index 74c713014a4..34d402e3672 100644
--- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/OneIssuePerLineSensor.java
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/OneIssuePerLineSensor.java
@@ -45,6 +45,7 @@ public class OneIssuePerLineSensor implements Sensor {
descriptor
.name("One Issue Per Line")
.onlyOnLanguages(Xoo.KEY, Xoo2.KEY)
+ .onlyWhenConfiguration(c -> !c.getBoolean("sonar.markAsUnchanged").orElse(false))
.createIssuesForRuleRepositories(XooRulesDefinition.XOO_REPOSITORY, XooRulesDefinition.XOO2_REPOSITORY)
.processesFilesIndependently();
}
diff --git a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/MarkAsUnchangedSensorTest.java b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/MarkAsUnchangedSensorTest.java
new file mode 100644
index 00000000000..2b9af3daee3
--- /dev/null
+++ b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/MarkAsUnchangedSensorTest.java
@@ -0,0 +1,74 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.xoo.rule;
+
+import java.io.IOException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
+import org.sonar.api.batch.sensor.internal.DefaultSensorDescriptor;
+import org.sonar.api.batch.sensor.internal.SensorContextTester;
+import org.sonar.api.config.Configuration;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.xoo.Xoo;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class MarkAsUnchangedSensorTest {
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+ private final MarkAsUnchangedSensor sensor = new MarkAsUnchangedSensor();
+
+ @Test
+ public void mark_as_unchanged_for_all_files() throws IOException {
+ SensorContextTester context = SensorContextTester.create(temp.newFolder());
+ DefaultInputFile inputFile1 = createFile("file1");
+ DefaultInputFile inputFile2 = createFile("file2");
+
+ context.fileSystem()
+ .add(inputFile1)
+ .add(inputFile2);
+
+ sensor.execute(context);
+ assertThat(inputFile1.isMarkedAsUnchanged()).isTrue();
+ assertThat(inputFile2.isMarkedAsUnchanged()).isTrue();
+ }
+
+ @Test
+ public void only_runs_if_property_is_set() {
+ DefaultSensorDescriptor descriptor = new DefaultSensorDescriptor();
+ sensor.describe(descriptor);
+ Configuration configWithProperty = new MapSettings().setProperty("sonar.markAsUnchanged", "true").asConfig();
+ Configuration configWithoutProperty = new MapSettings().asConfig();
+
+ assertThat(descriptor.configurationPredicate().test(configWithoutProperty)).isFalse();
+ assertThat(descriptor.configurationPredicate().test(configWithProperty)).isTrue();
+ }
+
+ private DefaultInputFile createFile(String name) {
+ return new TestInputFileBuilder("foo", "src/" + name)
+ .setLanguage(Xoo.KEY)
+ .initMetadata("a\nb\nc\nd\ne\nf\ng\nh\ni\n")
+ .build();
+ }
+
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java
index 1c6bf0aff1a..6808c7b6db8 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java
@@ -345,7 +345,8 @@ public class ComponentTreeBuilder {
return new FileAttributes(
component.getIsTest(),
lang != null ? lang.intern() : null,
- component.getLines());
+ component.getLines(),
+ component.getMarkedAsUnchanged());
}
private static class Node {
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java
index b523627c0dd..c1c868ec1f9 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java
@@ -33,15 +33,25 @@ public class FileAttributes {
private final boolean unitTest;
@CheckForNull
private final String languageKey;
+ private final boolean markedAsUnchanged;
private final int lines;
public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines) {
+ this(unitTest, languageKey, lines, false);
+ }
+
+ public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines, boolean markedAsUnchanged) {
this.unitTest = unitTest;
this.languageKey = languageKey;
+ this.markedAsUnchanged = markedAsUnchanged;
checkArgument(lines > 0, "Number of lines must be greater than zero");
this.lines = lines;
}
+ public boolean isMarkedAsUnchanged() {
+ return markedAsUnchanged;
+ }
+
public boolean isUnitTest() {
return unitTest;
}
@@ -64,6 +74,7 @@ public class FileAttributes {
"languageKey='" + languageKey + '\'' +
", unitTest=" + unitTest +
", lines=" + lines +
+ ", markedAsUnchanged=" + markedAsUnchanged +
'}';
}
}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java
new file mode 100644
index 00000000000..ab962b6dfc8
--- /dev/null
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java
@@ -0,0 +1,30 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.component;
+
+public interface FileStatuses {
+ /**
+ * A file is unchanged compared to the last analysis if it was detected as unchanged by the scanner and
+ * it's confirmed to be unchanged by the CE, by comparing file hashes.
+ */
+ boolean isUnchanged(Component component);
+
+ boolean isDataUnchanged(Component component);
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java
new file mode 100644
index 00000000000..045b7ffa14e
--- /dev/null
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java
@@ -0,0 +1,98 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.component;
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
+import org.sonar.ce.task.projectanalysis.source.SourceHashRepository;
+import org.sonar.db.source.FileHashesDto;
+
+import static com.google.common.base.Preconditions.checkState;
+import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER;
+
+public class FileStatusesImpl implements FileStatuses {
+ private final PreviousSourceHashRepository previousSourceHashRepository;
+ private final SourceHashRepository sourceHashRepository;
+ private final AnalysisMetadataHolder analysisMetadataHolder;
+ private final TreeRootHolder treeRootHolder;
+ private Set<String> fileUuidsMarkedAsUnchanged;
+
+ public FileStatusesImpl(AnalysisMetadataHolder analysisMetadataHolder, TreeRootHolder treeRootHolder, PreviousSourceHashRepository previousSourceHashRepository,
+ SourceHashRepository sourceHashRepository) {
+ this.analysisMetadataHolder = analysisMetadataHolder;
+ this.treeRootHolder = treeRootHolder;
+ this.previousSourceHashRepository = previousSourceHashRepository;
+ this.sourceHashRepository = sourceHashRepository;
+ }
+
+ public void initialize() {
+ fileUuidsMarkedAsUnchanged = new HashSet<>();
+ if (!analysisMetadataHolder.isPullRequest() && !analysisMetadataHolder.isFirstAnalysis()) {
+ new DepthTraversalTypeAwareCrawler(new Visitor()).visit(treeRootHolder.getRoot());
+ }
+ }
+
+ private class Visitor extends TypeAwareVisitorAdapter {
+ private boolean canTrustUnchangedFlags = true;
+
+ private Visitor() {
+ super(CrawlerDepthLimit.FILE, PRE_ORDER);
+ }
+
+ @Override
+ public void visitFile(Component file) {
+ if (file.getStatus() != Component.Status.SAME || !canTrustUnchangedFlags) {
+ return;
+ }
+
+ canTrustUnchangedFlags = hashEquals(file);
+ if (canTrustUnchangedFlags) {
+ if (file.getFileAttributes().isMarkedAsUnchanged()) {
+ fileUuidsMarkedAsUnchanged.add(file.getUuid());
+ }
+ } else {
+ fileUuidsMarkedAsUnchanged.clear();
+ }
+ }
+ }
+
+ @Override
+ public boolean isUnchanged(Component component) {
+ failIfNotInitialized();
+ return component.getStatus() == Component.Status.SAME && hashEquals(component);
+ }
+
+ @Override
+ public boolean isDataUnchanged(Component component) {
+ failIfNotInitialized();
+ return fileUuidsMarkedAsUnchanged.contains(component.getUuid());
+ }
+
+ private boolean hashEquals(Component component) {
+ Optional<String> dbHash = previousSourceHashRepository.getDbFile(component).map(FileHashesDto::getSrcHash);
+ return dbHash.map(hash -> hash.equals(sourceHashRepository.getRawSourceHash(component))).orElse(false);
+ }
+
+ private void failIfNotInitialized() {
+ checkState(fileUuidsMarkedAsUnchanged != null, "Not initialized");
+ }
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepository.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepository.java
new file mode 100644
index 00000000000..1eb4b72c5d8
--- /dev/null
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepository.java
@@ -0,0 +1,27 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.component;
+
+import java.util.Optional;
+import org.sonar.db.source.FileHashesDto;
+
+public interface PreviousSourceHashRepository {
+ Optional<FileHashesDto> getDbFile(Component component);
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImpl.java
new file mode 100644
index 00000000000..e13d01293db
--- /dev/null
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImpl.java
@@ -0,0 +1,48 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.component;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import org.sonar.db.source.FileHashesDto;
+
+import static org.sonar.api.utils.Preconditions.checkNotNull;
+import static org.sonar.api.utils.Preconditions.checkState;
+
+public class PreviousSourceHashRepositoryImpl implements PreviousSourceHashRepository {
+ private Map<String, FileHashesDto> previousFileHashesByUuid = null;
+
+ public void set(Map<String, FileHashesDto> previousFileHashesByUuid) {
+ checkState(this.previousFileHashesByUuid == null, "Repository already initialized");
+ checkNotNull(previousFileHashesByUuid);
+ this.previousFileHashesByUuid = Collections.unmodifiableMap(previousFileHashesByUuid);
+ }
+
+ public Map<String, FileHashesDto> getMap() {
+ return previousFileHashesByUuid;
+ }
+
+ @Override
+ public Optional<FileHashesDto> getDbFile(Component component) {
+ checkState(previousFileHashesByUuid != null, "Repository not initialized");
+ return Optional.ofNullable(previousFileHashesByUuid.get(component.getUuid()));
+ }
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java
index c8045d67c54..89f249a57e2 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java
@@ -34,6 +34,8 @@ import org.sonar.ce.task.projectanalysis.component.BranchLoader;
import org.sonar.ce.task.projectanalysis.component.BranchPersisterImpl;
import org.sonar.ce.task.projectanalysis.component.ConfigurationRepositoryImpl;
import org.sonar.ce.task.projectanalysis.component.DisabledComponentsHolderImpl;
+import org.sonar.ce.task.projectanalysis.component.FileStatusesImpl;
+import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepositoryImpl;
import org.sonar.ce.task.projectanalysis.component.ProjectPersister;
import org.sonar.ce.task.projectanalysis.component.ReferenceBranchComponentUuids;
import org.sonar.ce.task.projectanalysis.component.ReportModulesPath;
@@ -193,6 +195,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop
new ComputationTempFolderProvider(),
ReportModulesPath.class,
+ FileStatusesImpl.class,
new MetricModule(),
// holders
@@ -213,6 +216,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop
SiblingComponentsWithOpenIssues.class,
// repositories
+ PreviousSourceHashRepositoryImpl.class,
LanguageRepositoryImpl.class,
MeasureRepositoryImpl.class,
EventRepositoryImpl.class,
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitor.java
index 5a322ab7ca1..8f4c2405d82 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitor.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitor.java
@@ -19,11 +19,13 @@
*/
package org.sonar.ce.task.projectanalysis.issue;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.sonar.ce.task.projectanalysis.component.Component;
import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit;
+import org.sonar.ce.task.projectanalysis.component.FileStatuses;
import org.sonar.ce.task.projectanalysis.component.ReferenceBranchComponentUuids;
import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter;
import org.sonar.ce.task.projectanalysis.util.cache.DiskCache.CacheAppender;
@@ -37,49 +39,68 @@ public class IntegrateIssuesVisitor extends TypeAwareVisitorAdapter {
private final ProtoIssueCache protoIssueCache;
private final TrackerRawInputFactory rawInputFactory;
+ private final TrackerBaseInputFactory baseInputFactory;
private final IssueLifecycle issueLifecycle;
private final IssueVisitors issueVisitors;
private final IssueTrackingDelegator issueTracking;
private final SiblingsIssueMerger issueStatusCopier;
private final ReferenceBranchComponentUuids referenceBranchComponentUuids;
private final PullRequestSourceBranchMerger pullRequestSourceBranchMerger;
+ private final FileStatuses fileStatuses;
public IntegrateIssuesVisitor(
ProtoIssueCache protoIssueCache,
TrackerRawInputFactory rawInputFactory,
+ TrackerBaseInputFactory baseInputFactory,
IssueLifecycle issueLifecycle,
IssueVisitors issueVisitors,
IssueTrackingDelegator issueTracking,
SiblingsIssueMerger issueStatusCopier,
ReferenceBranchComponentUuids referenceBranchComponentUuids,
- PullRequestSourceBranchMerger pullRequestSourceBranchMerger) {
+ PullRequestSourceBranchMerger pullRequestSourceBranchMerger,
+ FileStatuses fileStatuses) {
super(CrawlerDepthLimit.FILE, POST_ORDER);
this.protoIssueCache = protoIssueCache;
this.rawInputFactory = rawInputFactory;
+ this.baseInputFactory = baseInputFactory;
this.issueLifecycle = issueLifecycle;
this.issueVisitors = issueVisitors;
this.issueTracking = issueTracking;
this.issueStatusCopier = issueStatusCopier;
this.referenceBranchComponentUuids = referenceBranchComponentUuids;
this.pullRequestSourceBranchMerger = pullRequestSourceBranchMerger;
+ this.fileStatuses = fileStatuses;
}
@Override
public void visitAny(Component component) {
try (CacheAppender<DefaultIssue> cacheAppender = protoIssueCache.newAppender()) {
issueVisitors.beforeComponent(component);
- Input<DefaultIssue> rawInput = rawInputFactory.create(component);
- TrackingResult tracking = issueTracking.track(component, rawInput);
- fillNewOpenIssues(component, tracking.newIssues(), rawInput, cacheAppender);
- fillExistingOpenIssues(component, tracking.issuesToMerge(), cacheAppender);
- closeIssues(component, tracking.issuesToClose(), cacheAppender);
- copyIssues(component, tracking.issuesToCopy(), cacheAppender);
+
+ if (fileStatuses.isDataUnchanged(component)) {
+ // we assume there's a previous analysis of the same branch
+ Input<DefaultIssue> baseIssues = baseInputFactory.create(component);
+ useBaseIssues(component, baseIssues.getIssues(), cacheAppender);
+ } else {
+ Input<DefaultIssue> rawInput = rawInputFactory.create(component);
+ TrackingResult tracking = issueTracking.track(component, rawInput);
+ fillNewOpenIssues(component, tracking.newIssues(), rawInput, cacheAppender);
+ fillExistingOpenIssues(component, tracking.issuesToMerge(), cacheAppender);
+ closeIssues(component, tracking.issuesToClose(), cacheAppender);
+ copyIssues(component, tracking.issuesToCopy(), cacheAppender);
+ }
issueVisitors.afterComponent(component);
} catch (Exception e) {
throw new IllegalStateException(String.format("Fail to process issues of component '%s'", component.getDbKey()), e);
}
}
+ private void useBaseIssues(Component component, Collection<DefaultIssue> dbIssues, CacheAppender<DefaultIssue> cacheAppender) {
+ for (DefaultIssue issue : dbIssues) {
+ process(component, issue, cacheAppender);
+ }
+ }
+
private void fillNewOpenIssues(Component component, Stream<DefaultIssue> newIssues, Input<DefaultIssue> rawInput, CacheAppender<DefaultIssue> cacheAppender) {
List<DefaultIssue> newIssuesList = newIssues
.peek(issueLifecycle::initNewOpenIssue)
@@ -127,8 +148,7 @@ public class IntegrateIssuesVisitor extends TypeAwareVisitorAdapter {
private void process(Component component, DefaultIssue issue, CacheAppender<DefaultIssue> cacheAppender) {
issueLifecycle.doAutomaticTransition(issue);
issueVisitors.onIssue(component, issue);
- if (issue.isNew() || issue.isChanged() || issue.isCopied() ||
- issue.isNoLongerNewCodeReferenceIssue() || issue.isToBeMigratedAsNewCodeReferenceIssue()) {
+ if (issue.isNew() || issue.isChanged() || issue.isCopied() || issue.isNoLongerNewCodeReferenceIssue() || issue.isToBeMigratedAsNewCodeReferenceIssue()) {
cacheAppender.append(issue);
}
}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImpl.java
index 679fea498d4..609a3e942e9 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImpl.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImpl.java
@@ -28,8 +28,7 @@ import org.sonar.api.utils.log.Loggers;
import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
import org.sonar.ce.task.projectanalysis.batch.BatchReportReader;
import org.sonar.ce.task.projectanalysis.component.Component;
-import org.sonar.ce.task.projectanalysis.component.Component.Status;
-import org.sonar.ce.task.projectanalysis.source.SourceHashRepository;
+import org.sonar.ce.task.projectanalysis.component.FileStatuses;
import org.sonar.ce.task.projectanalysis.source.SourceLinesDiff;
import org.sonar.scanner.protocol.output.ScannerReport;
@@ -44,15 +43,15 @@ public class ScmInfoRepositoryImpl implements ScmInfoRepository {
private final ScmInfoDbLoader scmInfoDbLoader;
private final AnalysisMetadataHolder analysisMetadata;
private final SourceLinesDiff sourceLinesDiff;
- private final SourceHashRepository sourceHashRepository;
+ private final FileStatuses fileStatuses;
public ScmInfoRepositoryImpl(BatchReportReader scannerReportReader, AnalysisMetadataHolder analysisMetadata, ScmInfoDbLoader scmInfoDbLoader,
- SourceLinesDiff sourceLinesDiff, SourceHashRepository sourceHashRepository) {
+ SourceLinesDiff sourceLinesDiff, FileStatuses fileStatuses) {
this.scannerReportReader = scannerReportReader;
this.analysisMetadata = analysisMetadata;
this.scmInfoDbLoader = scmInfoDbLoader;
this.sourceLinesDiff = sourceLinesDiff;
- this.sourceHashRepository = sourceHashRepository;
+ this.fileStatuses = fileStatuses;
}
@Override
@@ -118,9 +117,7 @@ public class ScmInfoRepositoryImpl implements ScmInfoRepository {
}
ScmInfo scmInfo = keepAuthorAndRevision ? dbInfoOpt.get() : removeAuthorAndRevision(dbInfoOpt.get());
- boolean fileUnchanged = file.getStatus() == Status.SAME && sourceHashRepository.getRawSourceHash(file).equals(dbInfoOpt.get().fileHash());
-
- if (fileUnchanged) {
+ if (fileStatuses.isUnchanged(file)) {
return Optional.of(scmInfo);
}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStep.java
index 66f6798ba02..7748b6206f3 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStep.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStep.java
@@ -19,10 +19,7 @@
*/
package org.sonar.ce.task.projectanalysis.source;
-import com.google.common.collect.ImmutableMap;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import org.apache.commons.codec.digest.DigestUtils;
@@ -31,6 +28,7 @@ import org.sonar.api.utils.System2;
import org.sonar.ce.task.projectanalysis.component.Component;
import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit;
import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler;
+import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepository;
import org.sonar.ce.task.projectanalysis.component.TreeRootHolder;
import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter;
import org.sonar.ce.task.projectanalysis.scm.Changeset;
@@ -39,6 +37,7 @@ import org.sonar.core.util.UuidFactory;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.protobuf.DbFileSources;
+import org.sonar.db.source.FileHashesDto;
import org.sonar.db.source.FileSourceDto;
import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER;
@@ -51,10 +50,11 @@ public class PersistFileSourcesStep implements ComputationStep {
private final FileSourceDataComputer fileSourceDataComputer;
private final FileSourceDataWarnings fileSourceDataWarnings;
private final UuidFactory uuidFactory;
+ private final PreviousSourceHashRepository previousSourceHashRepository;
public PersistFileSourcesStep(DbClient dbClient, System2 system2, TreeRootHolder treeRootHolder,
SourceLinesHashRepository sourceLinesHash, FileSourceDataComputer fileSourceDataComputer,
- FileSourceDataWarnings fileSourceDataWarnings, UuidFactory uuidFactory) {
+ FileSourceDataWarnings fileSourceDataWarnings, UuidFactory uuidFactory, PreviousSourceHashRepository previousSourceHashRepository) {
this.dbClient = dbClient;
this.system2 = system2;
this.treeRootHolder = treeRootHolder;
@@ -62,6 +62,7 @@ public class PersistFileSourcesStep implements ComputationStep {
this.fileSourceDataComputer = fileSourceDataComputer;
this.fileSourceDataWarnings = fileSourceDataWarnings;
this.uuidFactory = uuidFactory;
+ this.previousSourceHashRepository = previousSourceHashRepository;
}
@Override
@@ -77,8 +78,6 @@ public class PersistFileSourcesStep implements ComputationStep {
private class FileSourceVisitor extends TypeAwareVisitorAdapter {
private final DbSession session;
-
- private Map<String, FileSourceDto> previousFileSourcesByUuid = new HashMap<>();
private String projectUuid;
private FileSourceVisitor(DbSession session) {
@@ -89,11 +88,6 @@ public class PersistFileSourcesStep implements ComputationStep {
@Override
public void visitProject(Component project) {
this.projectUuid = project.getUuid();
- session.select("org.sonar.db.source.FileSourceMapper.selectHashesForProject", ImmutableMap.of("projectUuid", projectUuid),
- context -> {
- FileSourceDto dto = (FileSourceDto) context.getResultObject();
- previousFileSourcesByUuid.put(dto.getFileUuid(), dto);
- });
}
@Override
@@ -115,7 +109,7 @@ public class PersistFileSourcesStep implements ComputationStep {
List<String> lineHashes = fileSourceData.getLineHashes();
Changeset latestChangeWithRevision = fileSourceData.getLatestChangeWithRevision();
int lineHashesVersion = sourceLinesHash.getLineHashesVersion(file);
- FileSourceDto previousDto = previousFileSourcesByUuid.get(file.getUuid());
+ FileHashesDto previousDto = previousSourceHashRepository.getDbFile(file).orElse(null);
if (previousDto == null) {
FileSourceDto dto = new FileSourceDto()
.setUuid(uuidFactory.create())
@@ -139,7 +133,8 @@ public class PersistFileSourcesStep implements ComputationStep {
boolean revisionUpdated = !ObjectUtils.equals(revision, previousDto.getRevision());
boolean lineHashesVersionUpdated = previousDto.getLineHashesVersion() != lineHashesVersion;
if (binaryDataUpdated || srcHashUpdated || revisionUpdated || lineHashesVersionUpdated) {
- previousDto
+ FileSourceDto updatedDto = new FileSourceDto()
+ .setUuid(previousDto.getUuid())
.setBinaryData(binaryData)
.setDataHash(dataHash)
.setSrcHash(srcHash)
@@ -147,7 +142,7 @@ public class PersistFileSourcesStep implements ComputationStep {
.setLineHashesVersion(lineHashesVersion)
.setRevision(revision)
.setUpdatedAt(system2.now());
- dbClient.fileSourceDao().update(session, previousDto);
+ dbClient.fileSourceDao().update(session, updatedDto);
session.commit();
}
}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStep.java
new file mode 100644
index 00000000000..22e885088ad
--- /dev/null
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStep.java
@@ -0,0 +1,68 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.step;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.sonar.ce.task.projectanalysis.component.FileStatusesImpl;
+import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepositoryImpl;
+import org.sonar.ce.task.projectanalysis.component.TreeRootHolder;
+import org.sonar.ce.task.step.ComputationStep;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.source.FileHashesDto;
+import org.sonar.db.source.FileSourceDao;
+
+public class LoadFileHashesAndStatusStep implements ComputationStep {
+ private final DbClient dbClient;
+ private final PreviousSourceHashRepositoryImpl previousFileHashesRepository;
+ private final FileStatusesImpl fileStatuses;
+ private final FileSourceDao fileSourceDao;
+ private final TreeRootHolder treeRootHolder;
+
+ public LoadFileHashesAndStatusStep(DbClient dbClient, PreviousSourceHashRepositoryImpl previousFileHashesRepository,
+ FileStatusesImpl fileStatuses, FileSourceDao fileSourceDao, TreeRootHolder treeRootHolder) {
+ this.dbClient = dbClient;
+ this.previousFileHashesRepository = previousFileHashesRepository;
+ this.fileStatuses = fileStatuses;
+ this.fileSourceDao = fileSourceDao;
+ this.treeRootHolder = treeRootHolder;
+ }
+
+ @Override
+ public void execute(Context context) {
+ Map<String, FileHashesDto> previousFileHashesByUuid = new HashMap<>();
+ String projectUuid = treeRootHolder.getRoot().getUuid();
+
+ try (DbSession session = dbClient.openSession(false)) {
+ fileSourceDao.scrollFileHashesByProjectUuid(session, projectUuid, ctx -> {
+ FileHashesDto dto = ctx.getResultObject();
+ previousFileHashesByUuid.put(dto.getFileUuid(), dto);
+ });
+ }
+ previousFileHashesRepository.set(previousFileHashesByUuid);
+ fileStatuses.initialize();
+ }
+
+ @Override
+ public String getDescription() {
+ return "Load file hashes and statuses";
+ }
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java
index f21a9e73a80..e9dd25a1df1 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java
@@ -22,12 +22,14 @@ package org.sonar.ce.task.projectanalysis.step;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import javax.annotation.Nonnull;
import org.sonar.ce.task.projectanalysis.component.Component;
import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit;
import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler;
+import org.sonar.ce.task.projectanalysis.component.FileStatuses;
import org.sonar.ce.task.projectanalysis.component.TreeRootHolder;
import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter;
import org.sonar.ce.task.projectanalysis.measure.BestValueOptimization;
@@ -41,8 +43,55 @@ import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.measure.LiveMeasureDto;
+import static org.sonar.api.measures.CoreMetrics.BLOCKER_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.BUGS_KEY;
+import static org.sonar.api.measures.CoreMetrics.CLASSES_KEY;
+import static org.sonar.api.measures.CoreMetrics.CLASS_COMPLEXITY_KEY;
+import static org.sonar.api.measures.CoreMetrics.CODE_SMELLS_KEY;
+import static org.sonar.api.measures.CoreMetrics.COGNITIVE_COMPLEXITY_KEY;
+import static org.sonar.api.measures.CoreMetrics.COMMENT_LINES_DENSITY_KEY;
+import static org.sonar.api.measures.CoreMetrics.COMMENT_LINES_KEY;
+import static org.sonar.api.measures.CoreMetrics.COMPLEXITY_IN_CLASSES_KEY;
+import static org.sonar.api.measures.CoreMetrics.COMPLEXITY_IN_FUNCTIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.COMPLEXITY_KEY;
+import static org.sonar.api.measures.CoreMetrics.CONFIRMED_ISSUES_KEY;
+import static org.sonar.api.measures.CoreMetrics.CRITICAL_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.DEVELOPMENT_COST_KEY;
+import static org.sonar.api.measures.CoreMetrics.EFFORT_TO_REACH_MAINTAINABILITY_RATING_A_KEY;
+import static org.sonar.api.measures.CoreMetrics.FALSE_POSITIVE_ISSUES_KEY;
+import static org.sonar.api.measures.CoreMetrics.FILES_KEY;
import static org.sonar.api.measures.CoreMetrics.FILE_COMPLEXITY_DISTRIBUTION_KEY;
+import static org.sonar.api.measures.CoreMetrics.FILE_COMPLEXITY_KEY;
+import static org.sonar.api.measures.CoreMetrics.FUNCTIONS_KEY;
import static org.sonar.api.measures.CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY;
+import static org.sonar.api.measures.CoreMetrics.FUNCTION_COMPLEXITY_KEY;
+import static org.sonar.api.measures.CoreMetrics.GENERATED_LINES_KEY;
+import static org.sonar.api.measures.CoreMetrics.GENERATED_NCLOC_KEY;
+import static org.sonar.api.measures.CoreMetrics.INFO_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
+import static org.sonar.api.measures.CoreMetrics.MAJOR_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.MINOR_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.NCLOC_DATA_KEY;
+import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
+import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY;
+import static org.sonar.api.measures.CoreMetrics.OPEN_ISSUES_KEY;
+import static org.sonar.api.measures.CoreMetrics.RELIABILITY_RATING_KEY;
+import static org.sonar.api.measures.CoreMetrics.RELIABILITY_REMEDIATION_EFFORT_KEY;
+import static org.sonar.api.measures.CoreMetrics.REOPENED_ISSUES_KEY;
+import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_KEY;
+import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_REVIEWED_KEY;
+import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_REVIEWED_STATUS_KEY;
+import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_TO_REVIEW_STATUS_KEY;
+import static org.sonar.api.measures.CoreMetrics.SECURITY_RATING_KEY;
+import static org.sonar.api.measures.CoreMetrics.SECURITY_REMEDIATION_EFFORT_KEY;
+import static org.sonar.api.measures.CoreMetrics.SECURITY_REVIEW_RATING_KEY;
+import static org.sonar.api.measures.CoreMetrics.SQALE_DEBT_RATIO_KEY;
+import static org.sonar.api.measures.CoreMetrics.SQALE_RATING_KEY;
+import static org.sonar.api.measures.CoreMetrics.STATEMENTS_KEY;
+import static org.sonar.api.measures.CoreMetrics.TECHNICAL_DEBT_KEY;
+import static org.sonar.api.measures.CoreMetrics.VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.VULNERABILITIES_KEY;
+import static org.sonar.api.measures.CoreMetrics.WONT_FIX_ISSUES_KEY;
import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER;
public class PersistLiveMeasuresStep implements ComputationStep {
@@ -51,20 +100,31 @@ public class PersistLiveMeasuresStep implements ComputationStep {
* List of metrics that should not be persisted on file measure.
*/
private static final Set<String> NOT_TO_PERSIST_ON_FILE_METRIC_KEYS = Set.of(FILE_COMPLEXITY_DISTRIBUTION_KEY, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY);
-
+ private static final Set<String> NOT_TO_PERSIST_ON_UNCHANGED_FILES = Set.of(
+ BLOCKER_VIOLATIONS_KEY, BUGS_KEY, CLASS_COMPLEXITY_KEY, CLASSES_KEY, CODE_SMELLS_KEY, COGNITIVE_COMPLEXITY_KEY, COMMENT_LINES_KEY, COMMENT_LINES_DENSITY_KEY,
+ COMPLEXITY_KEY, COMPLEXITY_IN_CLASSES_KEY, COMPLEXITY_IN_FUNCTIONS_KEY, CONFIRMED_ISSUES_KEY, CRITICAL_VIOLATIONS_KEY, DEVELOPMENT_COST_KEY,
+ EFFORT_TO_REACH_MAINTAINABILITY_RATING_A_KEY, FALSE_POSITIVE_ISSUES_KEY, FILE_COMPLEXITY_KEY, FILE_COMPLEXITY_DISTRIBUTION_KEY, FILES_KEY, FUNCTION_COMPLEXITY_KEY,
+ FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, FUNCTIONS_KEY, GENERATED_LINES_KEY, GENERATED_NCLOC_KEY, INFO_VIOLATIONS_KEY, LINES_KEY,
+ MAJOR_VIOLATIONS_KEY, MINOR_VIOLATIONS_KEY, NCLOC_KEY, NCLOC_DATA_KEY, NCLOC_LANGUAGE_DISTRIBUTION_KEY, OPEN_ISSUES_KEY, RELIABILITY_RATING_KEY,
+ RELIABILITY_REMEDIATION_EFFORT_KEY, REOPENED_ISSUES_KEY, SECURITY_HOTSPOTS_KEY, SECURITY_HOTSPOTS_REVIEWED_KEY, SECURITY_HOTSPOTS_REVIEWED_STATUS_KEY,
+ SECURITY_HOTSPOTS_TO_REVIEW_STATUS_KEY, SECURITY_RATING_KEY, SECURITY_REMEDIATION_EFFORT_KEY, SECURITY_REVIEW_RATING_KEY, SQALE_DEBT_RATIO_KEY, TECHNICAL_DEBT_KEY,
+ SQALE_RATING_KEY, STATEMENTS_KEY, VIOLATIONS_KEY, VULNERABILITIES_KEY, WONT_FIX_ISSUES_KEY
+ );
private final DbClient dbClient;
private final MetricRepository metricRepository;
private final MeasureToMeasureDto measureToMeasureDto;
private final TreeRootHolder treeRootHolder;
private final MeasureRepository measureRepository;
+ private final Optional<FileStatuses> fileStatuses;
public PersistLiveMeasuresStep(DbClient dbClient, MetricRepository metricRepository, MeasureToMeasureDto measureToMeasureDto,
- TreeRootHolder treeRootHolder, MeasureRepository measureRepository) {
+ TreeRootHolder treeRootHolder, MeasureRepository measureRepository, Optional<FileStatuses> fileStatuses) {
this.dbClient = dbClient;
this.metricRepository = metricRepository;
this.measureToMeasureDto = measureToMeasureDto;
this.treeRootHolder = treeRootHolder;
this.measureRepository = measureRepository;
+ this.fileStatuses = fileStatuses;
}
@Override
@@ -99,11 +159,12 @@ public class PersistLiveMeasuresStep implements ComputationStep {
@Override
public void visitAny(Component component) {
List<String> metricUuids = new ArrayList<>();
+ List<String> keptMetricUuids = new ArrayList<>();
Map<String, Measure> measures = measureRepository.getRawMeasures(component);
List<LiveMeasureDto> dtos = new ArrayList<>();
for (Map.Entry<String, Measure> measuresByMetricKey : measures.entrySet()) {
String metricKey = measuresByMetricKey.getKey();
- if (NOT_TO_PERSIST_ON_FILE_METRIC_KEYS.contains(metricKey) && component.getType() == Component.Type.FILE) {
+ if (NOT_TO_PERSIST_ON_FILE_METRIC_KEYS.contains(metricKey) ) {
continue;
}
Metric metric = metricRepository.getByKey(metricKey);
@@ -112,27 +173,42 @@ public class PersistLiveMeasuresStep implements ComputationStep {
if (!NonEmptyMeasure.INSTANCE.test(m) || !notBestValueOptimized.test(m)) {
continue;
}
+ metricUuids.add(metric.getUuid());
+ if (shouldSkipMetricOnUnchangedFile(component, metricKey)) {
+ keptMetricUuids.add(metric.getUuid());
+ continue;
+ }
LiveMeasureDto lm = measureToMeasureDto.toLiveMeasureDto(m, metric, component);
dtos.add(lm);
- metricUuids.add(metric.getUuid());
}
+ List<String> excludedMetricUuids = supportUpsert ? metricUuids : keptMetricUuids;
+ deleteNonexistentMeasures(dbSession, component.getUuid(), excludedMetricUuids);
+ dtos.forEach(dto -> insertMeasureOptimally(dbSession, dto));
+
+ dbSession.commit();
+ insertsOrUpdates += dtos.size();
+ }
+
+ private void deleteNonexistentMeasures(DbSession dbSession, String componentUuid, List<String> excludedMetricUuids) {
+ // The measures that no longer exist on the component must be deleted, for example
+ // when the coverage on a file goes to the "best value" 100%.
+ // The measures on deleted components are deleted by the step PurgeDatastoresStep
+ dbClient.liveMeasureDao().deleteByComponentUuidExcludingMetricUuids(dbSession, componentUuid, excludedMetricUuids);
+ }
+
+ private void insertMeasureOptimally(DbSession dbSession, LiveMeasureDto dto) {
if (supportUpsert) {
- for (LiveMeasureDto dto : dtos) {
- dbClient.liveMeasureDao().upsert(dbSession, dto);
- }
- // The measures that no longer exist on the component must be deleted, for example
- // when the coverage on a file goes to the "best value" 100%.
- // The measures on deleted components are deleted by the step PurgeDatastoresStep
- dbClient.liveMeasureDao().deleteByComponentUuidExcludingMetricUuids(dbSession, component.getUuid(), metricUuids);
+ dbClient.liveMeasureDao().upsert(dbSession, dto);
} else {
- dbClient.liveMeasureDao().deleteByComponent(dbSession, component.getUuid());
- dtos.forEach(dto -> dbClient.liveMeasureDao().insert(dbSession, dto));
+ dbClient.liveMeasureDao().insert(dbSession, dto);
}
+ }
- dbSession.commit();
- insertsOrUpdates += dtos.size();
+ private boolean shouldSkipMetricOnUnchangedFile(Component component, String metricKey) {
+ return component.getType() == Component.Type.FILE && fileStatuses.isPresent() &&
+ fileStatuses.get().isDataUnchanged(component) && NOT_TO_PERSIST_ON_UNCHANGED_FILES.contains(metricKey);
}
}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java
index de5442922b0..a67ed8c74d9 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java
@@ -50,6 +50,7 @@ public class ReportComputationSteps extends AbstractComputationSteps {
LoadQualityProfilesStep.class,
// load project related stuffs
+ LoadFileHashesAndStatusStep.class,
LoadQualityGateStep.class,
LoadPeriodsStep.class,
FileMoveDetectionStep.class,
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilderTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilderTest.java
index 80c02c98421..c5b115aa8a2 100644
--- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilderTest.java
+++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilderTest.java
@@ -553,6 +553,31 @@ public class ComponentTreeBuilderTest {
}
@Test
+ public void files_have_markedAsUnchanged_flag() {
+ ScannerReport.Component project = newBuilder()
+ .setType(PROJECT)
+ .setKey("c1")
+ .setRef(1)
+ .addChildRef(2)
+ .build();
+ scannerComponentProvider.add(newBuilder()
+ .setRef(2)
+ .setType(FILE)
+ .setMarkedAsUnchanged(true)
+ .setProjectRelativePath("src/js/Foo.js")
+ .setLines(1));
+
+ Component root = call(project);
+ assertThat(root.getUuid()).isEqualTo("generated_c1_uuid");
+
+ Component directory = root.getChildren().iterator().next();
+ assertThat(directory.getUuid()).isEqualTo("generated_c1:src/js_uuid");
+
+ Component file = directory.getChildren().iterator().next();
+ assertThat(file.getFileAttributes().isMarkedAsUnchanged()).isTrue();
+ }
+
+ @Test
public void issues_are_relocated_from_directories_and_modules_to_root() {
ScannerReport.Component project = newBuilder()
.setType(PROJECT)
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileAttributesTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileAttributesTest.java
index cc7075e8916..1e70547c45e 100644
--- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileAttributesTest.java
+++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileAttributesTest.java
@@ -25,29 +25,29 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
public class FileAttributesTest {
-
-
@Test
public void create_production_file() {
- FileAttributes underTest = new FileAttributes(true, "java", 10);
+ FileAttributes underTest = new FileAttributes(true, "java", 10, true);
assertThat(underTest.isUnitTest()).isTrue();
assertThat(underTest.getLanguageKey()).isEqualTo("java");
assertThat(underTest.getLines()).isEqualTo(10);
+ assertThat(underTest.isMarkedAsUnchanged()).isTrue();
}
@Test
public void create_unit_test() {
- FileAttributes underTest = new FileAttributes(true, "java", 10);
+ FileAttributes underTest = new FileAttributes(true, "java", 10, false);
assertThat(underTest.isUnitTest()).isTrue();
assertThat(underTest.getLanguageKey()).isEqualTo("java");
assertThat(underTest.getLines()).isEqualTo(10);
+ assertThat(underTest.isMarkedAsUnchanged()).isFalse();
}
@Test
public void create_without_language() {
- FileAttributes underTest = new FileAttributes(true, null, 10);
+ FileAttributes underTest = new FileAttributes(true, null, 10, false);
assertThat(underTest.isUnitTest()).isTrue();
assertThat(underTest.getLanguageKey()).isNull();
@@ -56,21 +56,23 @@ public class FileAttributesTest {
@Test
public void fail_with_IAE_when_lines_is_0() {
- assertThatThrownBy(() -> new FileAttributes(true, "java", 0))
+ assertThatThrownBy(() -> new FileAttributes(true, "java", 0, false))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Number of lines must be greater than zero");
}
@Test
public void fail_with_IAE_when_lines_is_less_than_0() {
- assertThatThrownBy(() -> new FileAttributes(true, "java", -10))
+ assertThatThrownBy(() -> new FileAttributes(true, "java", -10, false))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Number of lines must be greater than zero");
}
@Test
public void test_toString() {
- assertThat(new FileAttributes(true, "java", 10)).hasToString("FileAttributes{languageKey='java', unitTest=true, lines=10}");
- assertThat(new FileAttributes(false, null, 1)).hasToString("FileAttributes{languageKey='null', unitTest=false, lines=1}");
+ assertThat(new FileAttributes(true, "java", 10, true))
+ .hasToString("FileAttributes{languageKey='java', unitTest=true, lines=10, markedAsUnchanged=true}");
+ assertThat(new FileAttributes(false, null, 1, false))
+ .hasToString("FileAttributes{languageKey='null', unitTest=false, lines=1, markedAsUnchanged=false}");
}
}
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java
new file mode 100644
index 00000000000..3b181db1893
--- /dev/null
+++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java
@@ -0,0 +1,184 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.component;
+
+import java.util.Optional;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.ce.task.projectanalysis.analysis.Analysis;
+import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolderRule;
+import org.sonar.ce.task.projectanalysis.source.SourceHashRepository;
+import org.sonar.db.source.FileHashesDto;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class FileStatusesImplTest {
+ private static final String PROJECT_KEY = "PROJECT_KEY";
+ private static final String PROJECT_UUID = "UUID-1234";
+
+ @Rule
+ public final TreeRootHolderRule treeRootHolder = new TreeRootHolderRule();
+ private final PreviousSourceHashRepository previousSourceHashRepository = mock(PreviousSourceHashRepository.class);
+ private final SourceHashRepository sourceHashRepository = mock(SourceHashRepository.class);
+ @Rule
+ public final AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule();
+ private final FileStatusesImpl fileStatuses = new FileStatusesImpl(analysisMetadataHolder, treeRootHolder, previousSourceHashRepository, sourceHashRepository);
+
+ @Before
+ public void before() {
+ analysisMetadataHolder.setBaseAnalysis(new Analysis.Builder().setUuid(PROJECT_UUID).setCreatedAt(1000L).build());
+ }
+
+ @Test
+ public void file_is_unchanged_only_if_status_is_SAME_and_hashes_equal() {
+ Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME).build();
+ Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build();
+ Component file3 = ReportComponent.builder(Component.Type.FILE, 4, "FILE3_KEY").setStatus(Component.Status.CHANGED).build();
+
+ addDbFileHash(file1, "hash1");
+ addDbFileHash(file2, "different");
+ addDbFileHash(file3, "hash3");
+
+ addReportFileHash(file1, "hash1");
+ addReportFileHash(file2, "hash2");
+ addReportFileHash(file3, "hash3");
+
+ Component project = ReportComponent.builder(Component.Type.PROJECT, 1)
+ .setUuid(PROJECT_UUID)
+ .setKey(PROJECT_KEY)
+ .addChildren(file1, file2, file3)
+ .build();
+ treeRootHolder.setRoot(project);
+ fileStatuses.initialize();
+ assertThat(fileStatuses.isUnchanged(file1)).isTrue();
+ assertThat(fileStatuses.isUnchanged(file2)).isFalse();
+ assertThat(fileStatuses.isUnchanged(file2)).isFalse();
+ }
+
+ @Test
+ public void isDataUnchanged_returns_false_if_any_SAME_status_is_incorrect() {
+ Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME)
+ .setFileAttributes(new FileAttributes(false, null, 10, true)).build();
+ Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build();
+
+ addDbFileHash(file1, "hash1");
+ addDbFileHash(file2, "different");
+
+ addReportFileHash(file1, "hash1");
+ addReportFileHash(file2, "hash2");
+
+ Component project = ReportComponent.builder(Component.Type.PROJECT, 1)
+ .setUuid(PROJECT_UUID)
+ .setKey(PROJECT_KEY)
+ .addChildren(file1, file2)
+ .build();
+ treeRootHolder.setRoot(project);
+ fileStatuses.initialize();
+ assertThat(fileStatuses.isDataUnchanged(file1)).isFalse();
+ assertThat(fileStatuses.isDataUnchanged(file2)).isFalse();
+ }
+
+ @Test
+ public void isDataUnchanged_returns_false_no_previous_analysis() {
+ analysisMetadataHolder.setBaseAnalysis(null);
+
+ Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME)
+ .setFileAttributes(new FileAttributes(false, null, 10, true)).build();
+ Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build();
+
+ addReportFileHash(file1, "hash1");
+ addReportFileHash(file2, "hash2");
+
+ Component project = ReportComponent.builder(Component.Type.PROJECT, 1)
+ .setUuid(PROJECT_UUID)
+ .setKey(PROJECT_KEY)
+ .addChildren(file1, file2)
+ .build();
+ treeRootHolder.setRoot(project);
+ fileStatuses.initialize();
+
+ assertThat(fileStatuses.isDataUnchanged(file1)).isFalse();
+ assertThat(fileStatuses.isDataUnchanged(file2)).isFalse();
+ }
+
+ @Test
+ public void isDataUnchanged_returns_false_if_not_set_by_analyzer() {
+ Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME)
+ .setFileAttributes(new FileAttributes(false, null, 10, false)).build();
+ Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build();
+
+ addDbFileHash(file1, "hash1");
+ addDbFileHash(file2, "hash2");
+
+ addReportFileHash(file1, "hash1");
+ addReportFileHash(file2, "hash2");
+
+ Component project = ReportComponent.builder(Component.Type.PROJECT, 1)
+ .setUuid(PROJECT_UUID)
+ .setKey(PROJECT_KEY)
+ .addChildren(file1, file2)
+ .build();
+ treeRootHolder.setRoot(project);
+ fileStatuses.initialize();
+ assertThat(fileStatuses.isDataUnchanged(file1)).isFalse();
+ assertThat(fileStatuses.isDataUnchanged(file2)).isFalse();
+ }
+
+ @Test
+ public void isDataUnchanged_returns_true_if_set_by_analyzer_and_all_SAME_status_are_correct() {
+ Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME)
+ .setFileAttributes(new FileAttributes(false, null, 10, true)).build();
+ Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build();
+ Component file3 = ReportComponent.builder(Component.Type.FILE, 4, "FILE3_KEY").setStatus(Component.Status.CHANGED).build();
+
+ addDbFileHash(file1, "hash1");
+ addDbFileHash(file2, "hash2");
+ addDbFileHash(file3, "hash3");
+
+ addReportFileHash(file1, "hash1");
+ addReportFileHash(file2, "hash2");
+ addReportFileHash(file3, "different");
+
+ Component project = ReportComponent.builder(Component.Type.PROJECT, 1)
+ .setUuid(PROJECT_UUID)
+ .setKey(PROJECT_KEY)
+ .addChildren(file1, file2, file3)
+ .build();
+ treeRootHolder.setRoot(project);
+ fileStatuses.initialize();
+ assertThat(fileStatuses.isDataUnchanged(file1)).isTrue();
+ assertThat(fileStatuses.isDataUnchanged(file2)).isFalse();
+
+ verify(previousSourceHashRepository).getDbFile(file1);
+ }
+
+ private void addDbFileHash(Component file, String hash) {
+ FileHashesDto fileHashesDto = new FileHashesDto().setSrcHash(hash);
+ when(previousSourceHashRepository.getDbFile(file)).thenReturn(Optional.of(fileHashesDto));
+ }
+
+ private void addReportFileHash(Component file, String hash) {
+ when(sourceHashRepository.getRawSourceHash(file)).thenReturn(hash);
+ }
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImplTest.java
new file mode 100644
index 00000000000..7ff9771fd48
--- /dev/null
+++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImplTest.java
@@ -0,0 +1,62 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.component;
+
+import java.util.Map;
+import org.junit.Test;
+import org.sonar.db.source.FileHashesDto;
+import org.sonar.db.source.FileSourceDto;
+
+import static java.util.Collections.emptyMap;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
+
+public class PreviousSourceHashRepositoryImplTest {
+ private final PreviousSourceHashRepositoryImpl previousFileHashesRepository = new PreviousSourceHashRepositoryImpl();
+
+ @Test
+ public void return_file_hashes() {
+ Component file1 = ReportComponent.builder(Component.Type.FILE, 1).build();
+ Component file2 = ReportComponent.builder(Component.Type.FILE, 2).build();
+ Component file3 = ReportComponent.builder(Component.Type.FILE, 3).build();
+
+ FileSourceDto fileSource1 = new FileSourceDto();
+ FileSourceDto fileSource2 = new FileSourceDto();
+
+ previousFileHashesRepository.set(Map.of(file1.getUuid(), fileSource1, file2.getUuid(), fileSource2));
+ assertThat(previousFileHashesRepository.getDbFile(file1)).contains(fileSource1);
+ assertThat(previousFileHashesRepository.getDbFile(file2)).contains(fileSource2);
+ assertThat(previousFileHashesRepository.getDbFile(file3)).isEmpty();
+ }
+
+ @Test
+ public void fail_if_not_set() {
+ assertThatThrownBy(() -> previousFileHashesRepository.getDbFile(mock(Component.class))).isInstanceOf(IllegalStateException.class);
+ }
+
+ @Test
+ public void fail_if_set_twice() {
+ Map<String, FileHashesDto> empty = emptyMap();
+ previousFileHashesRepository.set(empty);
+ assertThatThrownBy(() -> previousFileHashesRepository.set(empty)).isInstanceOf(IllegalStateException.class);
+
+ }
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitorTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitorTest.java
index 2f3ebe623f3..7e261b7ad72 100644
--- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitorTest.java
+++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitorTest.java
@@ -36,6 +36,7 @@ import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
import org.sonar.ce.task.projectanalysis.analysis.Branch;
import org.sonar.ce.task.projectanalysis.batch.BatchReportReaderRule;
import org.sonar.ce.task.projectanalysis.component.Component;
+import org.sonar.ce.task.projectanalysis.component.FileStatuses;
import org.sonar.ce.task.projectanalysis.component.ReferenceBranchComponentUuids;
import org.sonar.ce.task.projectanalysis.component.ReportComponent;
import org.sonar.ce.task.projectanalysis.component.ReportModulesPath;
@@ -50,7 +51,6 @@ import org.sonar.ce.task.projectanalysis.qualityprofile.AlwaysActiveRulesHolderI
import org.sonar.ce.task.projectanalysis.source.NewLinesRepository;
import org.sonar.ce.task.projectanalysis.source.SourceLinesHashRepository;
import org.sonar.ce.task.projectanalysis.source.SourceLinesRepository;
-import org.sonar.ce.task.projectanalysis.source.SourceLinesRepositoryRule;
import org.sonar.core.issue.DefaultIssue;
import org.sonar.core.issue.FieldDiffs;
import org.sonar.core.issue.IssueChangeContext;
@@ -113,8 +113,6 @@ public class IntegrateIssuesVisitorTest {
public ActiveRulesHolderRule activeRulesHolderRule = new ActiveRulesHolderRule();
@Rule
public RuleRepositoryRule ruleRepositoryRule = new RuleRepositoryRule();
- @Rule
- public SourceLinesRepositoryRule fileSourceRepository = new SourceLinesRepositoryRule();
private final AnalysisMetadataHolder analysisMetadataHolder = mock(AnalysisMetadataHolder.class);
private final IssueFilter issueFilter = mock(IssueFilter.class);
@@ -128,8 +126,9 @@ public class IntegrateIssuesVisitorTest {
private final ReferenceBranchComponentUuids referenceBranchComponentUuids = mock(ReferenceBranchComponentUuids.class);
private final SourceLinesHashRepository sourceLinesHash = mock(SourceLinesHashRepository.class);
private final NewLinesRepository newLinesRepository = mock(NewLinesRepository.class);
- private TargetBranchComponentUuids targetBranchComponentUuids = mock(TargetBranchComponentUuids.class);
+ private final TargetBranchComponentUuids targetBranchComponentUuids = mock(TargetBranchComponentUuids.class);
private final SourceLinesRepository sourceLinesRepository = mock(SourceLinesRepository.class);
+ private final FileStatuses fileStatuses = mock(FileStatuses.class);
private ArgumentCaptor<DefaultIssue> defaultIssueCaptor;
private final ComponentIssuesLoader issuesLoader = new ComponentIssuesLoader(dbTester.getDbClient(), ruleRepositoryRule, activeRulesHolderRule, new MapSettings().asConfig(),
@@ -166,8 +165,8 @@ public class IntegrateIssuesVisitorTest {
protoIssueCache = new ProtoIssueCache(temp.newFile(), System2.INSTANCE);
when(issueFilter.accept(any(DefaultIssue.class), eq(FILE))).thenReturn(true);
when(issueChangeContext.date()).thenReturn(new Date());
- underTest = new IntegrateIssuesVisitor(protoIssueCache, rawInputFactory, issueLifecycle, issueVisitors, trackingDelegator, issueStatusCopier, referenceBranchComponentUuids,
- mock(PullRequestSourceBranchMerger.class));
+ underTest = new IntegrateIssuesVisitor(protoIssueCache, rawInputFactory, baseInputFactory, issueLifecycle, issueVisitors, trackingDelegator, issueStatusCopier,
+ referenceBranchComponentUuids, mock(PullRequestSourceBranchMerger.class), fileStatuses);
}
@Test
@@ -181,7 +180,6 @@ public class IntegrateIssuesVisitorTest {
.setSeverity(Constants.Severity.BLOCKER)
.build();
reportReader.putIssues(FILE_REF, singletonList(reportIssue));
- fileSourceRepository.addLine(FILE_REF, "line1");
underTest.visitAny(FILE);
@@ -190,7 +188,6 @@ public class IntegrateIssuesVisitorTest {
@Test
public void process_existing_issue() {
-
RuleKey ruleKey = RuleTesting.XOO_X1;
// Issue from db has severity major
addBaseIssue(ruleKey);
@@ -203,19 +200,16 @@ public class IntegrateIssuesVisitorTest {
.setSeverity(Constants.Severity.BLOCKER)
.build();
reportReader.putIssues(FILE_REF, singletonList(reportIssue));
- fileSourceRepository.addLine(FILE_REF, "line1");
underTest.visitAny(FILE);
List<DefaultIssue> issues = newArrayList(protoIssueCache.traverse());
assertThat(issues).hasSize(1);
assertThat(issues.get(0).severity()).isEqualTo(Severity.BLOCKER);
-
}
@Test
public void dont_cache_existing_issue_if_unmodified() {
-
RuleKey ruleKey = RuleTesting.XOO_X1;
// Issue from db has severity major
addBaseIssue(ruleKey);
@@ -228,14 +222,12 @@ public class IntegrateIssuesVisitorTest {
.setSeverity(Constants.Severity.BLOCKER)
.build();
reportReader.putIssues(FILE_REF, singletonList(reportIssue));
- fileSourceRepository.addLine(FILE_REF, "line1");
underTest.visitAny(FILE);
List<DefaultIssue> issues = newArrayList(protoIssueCache.traverse());
assertThat(issues).hasSize(1);
assertThat(issues.get(0).severity()).isEqualTo(Severity.BLOCKER);
-
}
@Test
@@ -248,7 +240,6 @@ public class IntegrateIssuesVisitorTest {
.setSeverity(Constants.Severity.BLOCKER)
.build();
reportReader.putIssues(FILE_REF, singletonList(reportIssue));
- fileSourceRepository.addLine(FILE_REF, "line1");
underTest.visitAny(FILE);
@@ -280,8 +271,34 @@ public class IntegrateIssuesVisitorTest {
}
@Test
- public void copy_issues_when_creating_new_non_main_branch() {
+ public void reuse_issues_when_data_unchanged() {
+ RuleKey ruleKey = RuleTesting.XOO_X1;
+ // Issue from db has severity major
+ addBaseIssue(ruleKey);
+
+ // Issue from report has severity blocker
+ ScannerReport.Issue reportIssue = ScannerReport.Issue.newBuilder()
+ .setMsg("new message")
+ .setRuleRepository(ruleKey.repository())
+ .setRuleKey(ruleKey.rule())
+ .setSeverity(Constants.Severity.BLOCKER)
+ .build();
+ reportReader.putIssues(FILE_REF, singletonList(reportIssue));
+ when(fileStatuses.isDataUnchanged(FILE)).thenReturn(true);
+
+ underTest.visitAny(FILE);
+ // visitors get called, so measures created from issues should be calculated taking these issues into account
+ verify(issueVisitor).onIssue(eq(FILE), defaultIssueCaptor.capture());
+ assertThat(defaultIssueCaptor.getValue().ruleKey().rule()).isEqualTo(ruleKey.rule());
+
+ // most issues won't go to the cache since they aren't changed and don't need to be persisted
+ // In this test they are being closed but the workflows aren't working (we mock them) so nothing is changed on the issue is not cached.
+ assertThat(newArrayList(protoIssueCache.traverse())).isEmpty();
+ }
+
+ @Test
+ public void copy_issues_when_creating_new_non_main_branch() {
when(mergeBranchComponentsUuids.getComponentUuid(FILE_KEY)).thenReturn(FILE_UUID_ON_BRANCH);
when(referenceBranchComponentUuids.getReferenceBranchName()).thenReturn("master");
@@ -304,7 +321,6 @@ public class IntegrateIssuesVisitorTest {
.setSeverity(Constants.Severity.BLOCKER)
.build();
reportReader.putIssues(FILE_REF, singletonList(reportIssue));
- fileSourceRepository.addLine(FILE_REF, "line1");
underTest.visitAny(FILE);
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java
index 16c69b83a34..300d34bc83c 100644
--- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java
+++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java
@@ -40,9 +40,9 @@ import org.sonar.ce.task.projectanalysis.component.Component;
import org.sonar.ce.task.projectanalysis.component.Component.Status;
import org.sonar.ce.task.projectanalysis.component.Component.Type;
import org.sonar.ce.task.projectanalysis.component.FileAttributes;
+import org.sonar.ce.task.projectanalysis.component.FileStatuses;
import org.sonar.ce.task.projectanalysis.component.ReportComponent;
import org.sonar.ce.task.projectanalysis.component.ViewsComponent;
-import org.sonar.ce.task.projectanalysis.source.SourceHashRepository;
import org.sonar.ce.task.projectanalysis.source.SourceLinesDiff;
import org.sonar.db.protobuf.DbFileSources.Line;
import org.sonar.scanner.protocol.output.ScannerReport;
@@ -74,12 +74,11 @@ public class ScmInfoRepositoryImplTest {
@Rule
public AnalysisMetadataHolderRule analysisMetadata = new AnalysisMetadataHolderRule();
- private SourceHashRepository sourceHashRepository = mock(SourceHashRepository.class);
- private SourceLinesDiff diff = mock(SourceLinesDiff.class);
- private ScmInfoDbLoader dbLoader = mock(ScmInfoDbLoader.class);
- private Date analysisDate = new Date();
-
- private ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(reportReader, analysisMetadata, dbLoader, diff, sourceHashRepository);
+ private final FileStatuses fileStatuses = mock(FileStatuses.class);
+ private final SourceLinesDiff diff = mock(SourceLinesDiff.class);
+ private final ScmInfoDbLoader dbLoader = mock(ScmInfoDbLoader.class);
+ private final Date analysisDate = new Date();
+ private final ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(reportReader, analysisMetadata, dbLoader, diff, fileStatuses);
@Before
public void setUp() {
@@ -106,7 +105,7 @@ public class ScmInfoRepositoryImplTest {
assertThat(logTester.logs(TRACE)).isEmpty();
verifyNoInteractions(dbLoader);
- verifyNoInteractions(sourceHashRepository);
+ verifyNoInteractions(fileStatuses);
verifyNoInteractions(diff);
}
@@ -125,32 +124,32 @@ public class ScmInfoRepositoryImplTest {
assertThat(logTester.logs(TRACE)).containsOnly("Reading SCM info from report for file 'FILE_KEY'");
verifyNoInteractions(dbLoader);
- verifyNoInteractions(sourceHashRepository);
+ verifyNoInteractions(fileStatuses);
verifyNoInteractions(diff);
}
@Test
public void read_from_DB_if_no_report_and_file_unchanged() {
- createDbScmInfoWithOneLine("hash");
- when(sourceHashRepository.getRawSourceHash(FILE_SAME)).thenReturn("hash");
+ createDbScmInfoWithOneLine();
+ when(fileStatuses.isUnchanged(FILE_SAME)).thenReturn(true);
// should clear revision and author
ScmInfo scmInfo = underTest.getScmInfo(FILE_SAME).get();
assertThat(scmInfo.getAllChangesets()).hasSize(1);
assertChangeset(scmInfo.getChangesetForLine(1), null, null, 10L);
- verify(sourceHashRepository).getRawSourceHash(FILE_SAME);
+ verify(fileStatuses).isUnchanged(FILE_SAME);
verify(dbLoader).getScmInfo(FILE_SAME);
verifyNoMoreInteractions(dbLoader);
- verifyNoMoreInteractions(sourceHashRepository);
+ verifyNoMoreInteractions(fileStatuses);
verifyNoInteractions(diff);
}
@Test
public void read_from_DB_if_no_report_and_file_unchanged_and_copyFromPrevious_is_true() {
- createDbScmInfoWithOneLine("hash");
- when(sourceHashRepository.getRawSourceHash(FILE_SAME)).thenReturn("hash");
+ createDbScmInfoWithOneLine();
+ when(fileStatuses.isUnchanged(FILE_SAME)).thenReturn(true);
addFileSourceInReport(1);
addCopyFromPrevious();
@@ -158,11 +157,11 @@ public class ScmInfoRepositoryImplTest {
assertThat(scmInfo.getAllChangesets()).hasSize(1);
assertChangeset(scmInfo.getChangesetForLine(1), "rev1", "author1", 10L);
- verify(sourceHashRepository).getRawSourceHash(FILE_SAME);
+ verify(fileStatuses).isUnchanged(FILE_SAME);
verify(dbLoader).getScmInfo(FILE_SAME);
verifyNoMoreInteractions(dbLoader);
- verifyNoMoreInteractions(sourceHashRepository);
+ verifyNoMoreInteractions(fileStatuses);
verifyNoInteractions(diff);
}
@@ -178,7 +177,7 @@ public class ScmInfoRepositoryImplTest {
verify(dbLoader).getScmInfo(FILE);
verifyNoMoreInteractions(dbLoader);
- verifyNoInteractions(sourceHashRepository);
+ verifyNoInteractions(fileStatuses);
verifyNoInteractions(diff);
}
@@ -195,13 +194,13 @@ public class ScmInfoRepositoryImplTest {
verify(dbLoader).getScmInfo(FILE);
verifyNoMoreInteractions(dbLoader);
- verifyNoInteractions(sourceHashRepository);
+ verifyNoInteractions(fileStatuses);
verifyNoInteractions(diff);
}
@Test
public void generate_scm_info_for_new_and_changed_lines_when_report_is_empty() {
- createDbScmInfoWithOneLine("hash");
+ createDbScmInfoWithOneLine();
when(diff.computeMatchingLines(FILE)).thenReturn(new int[] {1, 0, 0});
addFileSourceInReport(3);
ScmInfo scmInfo = underTest.getScmInfo(FILE).get();
@@ -214,7 +213,6 @@ public class ScmInfoRepositoryImplTest {
verify(dbLoader).getScmInfo(FILE);
verify(diff).computeMatchingLines(FILE);
verifyNoMoreInteractions(dbLoader);
- verifyNoInteractions(sourceHashRepository);
verifyNoMoreInteractions(diff);
}
@@ -229,7 +227,7 @@ public class ScmInfoRepositoryImplTest {
@UseDataProvider("allTypeComponentButFile")
public void do_not_query_db_nor_report_if_component_type_is_not_FILE(Component component) {
BatchReportReader batchReportReader = mock(BatchReportReader.class);
- ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(batchReportReader, analysisMetadata, dbLoader, diff, sourceHashRepository);
+ ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(batchReportReader, analysisMetadata, dbLoader, diff, fileStatuses);
assertThat(underTest.getScmInfo(component)).isEmpty();
@@ -278,13 +276,13 @@ public class ScmInfoRepositoryImplTest {
reportReader.putChangesets(Changesets.newBuilder().setComponentRef(FILE_REF).setCopyFromPrevious(true).build());
}
- private DbScmInfo createDbScmInfoWithOneLine(String hash) {
+ private DbScmInfo createDbScmInfoWithOneLine() {
Line line1 = Line.newBuilder().setLine(1)
.setScmRevision("rev1")
.setScmAuthor("author1")
.setScmDate(10L)
.build();
- DbScmInfo scmInfo = DbScmInfo.create(Collections.singletonList(line1), 1, hash).get();
+ DbScmInfo scmInfo = DbScmInfo.create(Collections.singletonList(line1), 1, "hash1").get();
when(dbLoader.getScmInfo(FILE)).thenReturn(Optional.of(scmInfo));
return scmInfo;
}
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStepTest.java
index c7234e89aa2..105c5d20226 100644
--- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStepTest.java
+++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStepTest.java
@@ -22,15 +22,16 @@ package org.sonar.ce.task.projectanalysis.source;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.function.Consumer;
import org.apache.commons.codec.digest.DigestUtils;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
-import org.mockito.Mockito;
import org.sonar.api.utils.System2;
import org.sonar.ce.task.projectanalysis.component.Component;
import org.sonar.ce.task.projectanalysis.component.FileAttributes;
+import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepository;
import org.sonar.ce.task.projectanalysis.component.ReportComponent;
import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule;
import org.sonar.ce.task.projectanalysis.scm.Changeset;
@@ -43,10 +44,12 @@ import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;
import org.sonar.db.protobuf.DbFileSources;
+import org.sonar.db.source.FileHashesDto;
import org.sonar.db.source.FileSourceDto;
import org.sonar.db.source.LineHashVersion;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -60,28 +63,30 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
private static final long NOW = 123456789L;
private static final long PAST = 15000L;
- private System2 system2 = mock(System2.class);
+ private final System2 system2 = mock(System2.class);
@Rule
public DbTester dbTester = DbTester.create(system2);
@Rule
public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule();
- private SourceLinesHashRepository sourceLinesHashRepository = mock(SourceLinesHashRepository.class);
- private SourceLinesHashRepositoryImpl.LineHashesComputer lineHashesComputer = mock(SourceLinesHashRepositoryImpl.LineHashesComputer.class);
- private FileSourceDataComputer fileSourceDataComputer = mock(FileSourceDataComputer.class);
- private FileSourceDataWarnings fileSourceDataWarnings = mock(FileSourceDataWarnings.class);
+ private final SourceLinesHashRepository sourceLinesHashRepository = mock(SourceLinesHashRepository.class);
+ private final SourceLinesHashRepositoryImpl.LineHashesComputer lineHashesComputer = mock(SourceLinesHashRepositoryImpl.LineHashesComputer.class);
+ private final FileSourceDataComputer fileSourceDataComputer = mock(FileSourceDataComputer.class);
+ private final FileSourceDataWarnings fileSourceDataWarnings = mock(FileSourceDataWarnings.class);
+ private final PreviousSourceHashRepository previousSourceHashRepository = mock(PreviousSourceHashRepository.class);
- private DbClient dbClient = dbTester.getDbClient();
- private DbSession session = dbTester.getSession();
+ private final DbClient dbClient = dbTester.getDbClient();
+ private final DbSession session = dbTester.getSession();
private PersistFileSourcesStep underTest;
@Before
public void setup() {
when(system2.now()).thenReturn(NOW);
- when(sourceLinesHashRepository.getLineHashesComputerToPersist(Mockito.any(Component.class))).thenReturn(lineHashesComputer);
- underTest = new PersistFileSourcesStep(dbClient, system2, treeRootHolder, sourceLinesHashRepository, fileSourceDataComputer, fileSourceDataWarnings, new SequenceUuidFactory());
+ when(sourceLinesHashRepository.getLineHashesComputerToPersist(any(Component.class))).thenReturn(lineHashesComputer);
+ underTest = new PersistFileSourcesStep(dbClient, system2, treeRootHolder, sourceLinesHashRepository, fileSourceDataComputer, fileSourceDataWarnings,
+ new SequenceUuidFactory(), previousSourceHashRepository);
initBasicReport(1);
}
@@ -305,6 +310,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
@Test
public void not_update_sources_when_nothing_has_changed() {
+ setPastAnalysisHashes();
dbClient.fileSourceDao().insert(dbTester.getSession(), createDto());
dbTester.getSession().commit();
@@ -326,7 +332,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
public void update_sources_when_source_updated() {
// Existing sources
long past = 150000L;
- dbClient.fileSourceDao().insert(dbTester.getSession(), new FileSourceDto()
+ FileSourceDto dbFileSources = new FileSourceDto()
.setUuid(Uuids.createFast())
.setProjectUuid(PROJECT_UUID)
.setFileUuid(FILE1_UUID)
@@ -341,8 +347,10 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
.build())
.setCreatedAt(past)
.setUpdatedAt(past)
- .setRevision("rev-0"));
+ .setRevision("rev-0");
+ dbClient.fileSourceDao().insert(dbTester.getSession(), dbFileSources);
dbTester.getSession().commit();
+ setPastAnalysisHashes(dbFileSources);
DbFileSources.Data newSourceData = DbFileSources.Data.newBuilder()
.addLines(DbFileSources.Line.newBuilder()
@@ -369,8 +377,10 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
@Test
public void update_sources_when_src_hash_is_missing() {
- dbClient.fileSourceDao().insert(dbTester.getSession(), createDto(dto -> dto.setSrcHash(null)));
+ FileSourceDto dbFileSources = createDto(dto -> dto.setSrcHash(null));
+ dbClient.fileSourceDao().insert(dbTester.getSession(), dbFileSources);
dbTester.getSession().commit();
+ setPastAnalysisHashes(dbFileSources);
DbFileSources.Data sourceData = DbFileSources.Data.newBuilder().build();
setComputedData(sourceData, Collections.singletonList("lineHash"), "newSourceHash", null);
@@ -394,8 +404,10 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
.build())
.build();
- dbClient.fileSourceDao().insert(dbTester.getSession(), createDto(dto -> dto.setRevision(null)));
+ FileSourceDto dbFileSources = createDto(dto -> dto.setRevision(null));
+ dbClient.fileSourceDao().insert(dbTester.getSession(), dbFileSources);
dbTester.getSession().commit();
+ setPastAnalysisHashes(dbFileSources);
Changeset changeset = Changeset.newChangesetBuilder().setDate(1L).setRevision("revision").build();
setComputedData(sourceData, Collections.singletonList("137f72c3708c6bd0de00a0e5a69c699b"), "29f25900140c94db38035128cb6de6a2", changeset);
@@ -436,6 +448,21 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
return dto;
}
+ private void setPastAnalysisHashes() {
+ DbFileSources.Data sourceData = DbFileSources.Data.newBuilder().build();
+ byte[] data = FileSourceDto.encodeSourceData(sourceData);
+ String dataHash = DigestUtils.md5Hex(data);
+ FileHashesDto fileHashesDto = new FileHashesDto()
+ .setSrcHash("sourceHash")
+ .setDataHash(dataHash)
+ .setRevision("rev-1");
+ setPastAnalysisHashes(fileHashesDto);
+ }
+
+ private void setPastAnalysisHashes(FileHashesDto fileHashesDto) {
+ when(previousSourceHashRepository.getDbFile(any(Component.class))).thenReturn(Optional.of(fileHashesDto));
+ }
+
private void setComputedData(DbFileSources.Data data, List<String> lineHashes, String sourceHash, Changeset latestChangeWithRevision) {
FileSourceDataComputer.Data computedData = new FileSourceDataComputer.Data(data, lineHashes, sourceHash, latestChangeWithRevision);
when(fileSourceDataComputer.compute(fileComponent().build(), fileSourceDataWarnings)).thenReturn(computedData);
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStepTest.java
new file mode 100644
index 00000000000..68bb25db7b1
--- /dev/null
+++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStepTest.java
@@ -0,0 +1,126 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.step;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
+import org.sonar.ce.task.projectanalysis.component.Component;
+import org.sonar.ce.task.projectanalysis.component.FileStatusesImpl;
+import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepositoryImpl;
+import org.sonar.ce.task.projectanalysis.component.ReportComponent;
+import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule;
+import org.sonar.ce.task.step.TestComputationStepContext;
+import org.sonar.db.DbTester;
+import org.sonar.db.component.ComponentDto;
+import org.sonar.db.component.ComponentTesting;
+import org.sonar.db.source.FileHashesDto;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class LoadFileHashesAndStatusStepTest {
+ @Rule
+ public DbTester db = DbTester.create();
+ @Rule
+ public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule();
+ public PreviousSourceHashRepositoryImpl previousFileHashesRepository = new PreviousSourceHashRepositoryImpl();
+ public FileStatusesImpl fileStatuses = mock(FileStatusesImpl.class);
+ public AnalysisMetadataHolder analysisMetadataHolder = mock(AnalysisMetadataHolder.class);
+
+ private final LoadFileHashesAndStatusStep underTest = new LoadFileHashesAndStatusStep(db.getDbClient(), previousFileHashesRepository, fileStatuses,
+ db.getDbClient().fileSourceDao(), treeRootHolder);
+
+ @Before
+ public void before() {
+ when(analysisMetadataHolder.isFirstAnalysis()).thenReturn(false);
+ }
+
+ @Test
+ public void initialized_file_statuses() {
+ Component project = ReportComponent.builder(Component.Type.PROJECT, 1, "project").build();
+ treeRootHolder.setRoot(project);
+ underTest.execute(new TestComputationStepContext());
+ verify(fileStatuses).initialize();
+ }
+
+ @Test
+ public void loads_file_hashes_for_project_branch() {
+ ComponentDto project1 = db.components().insertPublicProject();
+ ComponentDto project2 = db.components().insertPublicProject();
+
+ ComponentDto dbFile1 = db.components().insertComponent(ComponentTesting.newFileDto(project1));
+ ComponentDto dbFile2 = db.components().insertComponent(ComponentTesting.newFileDto(project1));
+
+ insertFileSources(dbFile1, dbFile2);
+
+ Component reportFile1 = ReportComponent.builder(Component.Type.FILE, 2, dbFile1.getKey()).setUuid(dbFile1.uuid()).build();
+ Component reportFile2 = ReportComponent.builder(Component.Type.FILE, 3, dbFile2.getKey()).setUuid(dbFile2.uuid()).build();
+ Component reportFile3 = ReportComponent.builder(Component.Type.FILE, 4, dbFile2.getKey()).build();
+
+ treeRootHolder.setRoot(ReportComponent.builder(Component.Type.PROJECT, 1, project1.getKey()).setUuid(project1.uuid())
+ .addChildren(reportFile1, reportFile2, reportFile3).build());
+ underTest.execute(new TestComputationStepContext());
+
+ assertThat(previousFileHashesRepository.getMap()).hasSize(2);
+ assertThat(previousFileHashesRepository.getDbFile(reportFile1).get())
+ .extracting(FileHashesDto::getSrcHash, FileHashesDto::getRevision, FileHashesDto::getDataHash)
+ .containsOnly("srcHash" + dbFile1.getKey(), "revision" + dbFile1.getKey(), "dataHash" + dbFile1.getKey());
+ assertThat(previousFileHashesRepository.getDbFile(reportFile2).get())
+ .extracting(FileHashesDto::getSrcHash, FileHashesDto::getRevision, FileHashesDto::getDataHash)
+ .containsOnly("srcHash" + dbFile2.getKey(), "revision" + dbFile2.getKey(), "dataHash" + dbFile2.getKey());
+ assertThat(previousFileHashesRepository.getDbFile(reportFile3)).isEmpty();
+ }
+
+ @Test
+ public void loads_high_number_of_files() {
+ ComponentDto project1 = db.components().insertPublicProject();
+ List<Component> files = new ArrayList<>(2000);
+
+ for (int i = 0; i < 2000; i++) {
+ ComponentDto dbFile = db.components().insertComponent(ComponentTesting.newFileDto(project1));
+ insertFileSources(dbFile);
+ files.add(ReportComponent.builder(Component.Type.FILE, 2, dbFile.getKey()).setUuid(dbFile.uuid()).build());
+ }
+
+ treeRootHolder.setRoot(ReportComponent.builder(Component.Type.PROJECT, 1, project1.getKey()).setUuid(project1.uuid())
+ .addChildren(files.toArray(Component[]::new)).build());
+ underTest.execute(new TestComputationStepContext());
+
+ assertThat(previousFileHashesRepository.getMap()).hasSize(2000);
+ for (Component file : files) {
+ assertThat(previousFileHashesRepository.getDbFile(file)).isPresent();
+ }
+ }
+
+ private void insertFileSources(ComponentDto... files) {
+ for (ComponentDto file : files) {
+ db.fileSources().insertFileSource(file, f -> f
+ .setSrcHash("srcHash" + file.getKey())
+ .setRevision("revision" + file.getKey())
+ .setDataHash("dataHash" + file.getKey()));
+ }
+ }
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java
index 47ca82a48ab..be8adf493fc 100644
--- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java
+++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java
@@ -27,6 +27,7 @@ import org.sonar.api.measures.Metric;
import org.sonar.api.utils.System2;
import org.sonar.ce.task.projectanalysis.analysis.MutableAnalysisMetadataHolderRule;
import org.sonar.ce.task.projectanalysis.component.Component;
+import org.sonar.ce.task.projectanalysis.component.FileStatuses;
import org.sonar.ce.task.projectanalysis.component.ReportComponent;
import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule;
import org.sonar.ce.task.projectanalysis.component.ViewsComponent;
@@ -44,6 +45,10 @@ import org.sonar.server.project.Project;
import static java.util.Collections.emptyList;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.sonar.api.measures.CoreMetrics.BUGS;
import static org.sonar.ce.task.projectanalysis.component.Component.Type.DIRECTORY;
import static org.sonar.ce.task.projectanalysis.component.Component.Type.FILE;
import static org.sonar.ce.task.projectanalysis.component.Component.Type.PROJECT;
@@ -78,7 +83,9 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
@Rule
public MutableAnalysisMetadataHolderRule analysisMetadataHolder = new MutableAnalysisMetadataHolderRule();
- private DbClient dbClient = db.getDbClient();
+ private final FileStatuses fileStatuses = mock(FileStatuses.class);
+ private final DbClient dbClient = db.getDbClient();
+ private final TestComputationStepContext context = new TestComputationStepContext();
@Before
public void setUp() {
@@ -86,9 +93,11 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
MetricDto intMetricDto = db.measures().insertMetric(m -> m.setKey(INT_METRIC.getKey()).setValueType(Metric.ValueType.INT.name()));
MetricDto bestValueMMetricDto = db.measures()
.insertMetric(m -> m.setKey(METRIC_WITH_BEST_VALUE.getKey()).setValueType(Metric.ValueType.INT.name()).setOptimizedBestValue(true).setBestValue(0.0));
+ MetricDto bugs = db.measures().insertMetric(m -> m.setKey(BUGS.getKey()));
metricRepository.add(stringMetricDto.getUuid(), STRING_METRIC);
metricRepository.add(intMetricDto.getUuid(), INT_METRIC);
metricRepository.add(bestValueMMetricDto.getUuid(), METRIC_WITH_BEST_VALUE);
+ metricRepository.add(bugs.getUuid(), BUGS);
}
@Test
@@ -100,7 +109,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("dir-value"));
measureRepository.addRawMeasure(REF_4, STRING_METRIC.getKey(), newMeasureBuilder().create("file-value"));
- TestComputationStepContext context = new TestComputationStepContext();
step().execute(context);
// all measures are persisted, from project to file
@@ -117,7 +125,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
measureRepository.addRawMeasure(REF_1, STRING_METRIC.getKey(), newMeasureBuilder().createNoValue());
measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().createNoValue());
- TestComputationStepContext context = new TestComputationStepContext();
step().execute(context);
assertThatMeasureIsNotPersisted("project-uuid", STRING_METRIC);
@@ -130,7 +137,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
prepareProject();
measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().setVariation(42.0).createNoValue());
- TestComputationStepContext context = new TestComputationStepContext();
step().execute(context);
LiveMeasureDto persistedMeasure = selectMeasure("project-uuid", INT_METRIC).get();
@@ -152,7 +158,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
measureRepository.addRawMeasure(REF_4, INT_METRIC.getKey(), newMeasureBuilder().create(42));
- TestComputationStepContext context = new TestComputationStepContext();
step().execute(context);
assertThatMeasureHasValue(measureOnFileInProject, 42);
@@ -173,7 +178,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
// file measure with metric best value -> do not persist
measureRepository.addRawMeasure(REF_4, METRIC_WITH_BEST_VALUE.getKey(), newMeasureBuilder().create(0));
- TestComputationStepContext context = new TestComputationStepContext();
step().execute(context);
assertThatMeasureDoesNotExist(oldMeasure);
@@ -182,6 +186,30 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
}
@Test
+ public void keep_measures_for_unchanged_files() {
+ prepareProject();
+ LiveMeasureDto oldMeasure = insertMeasure("file-uuid", "project-uuid", BUGS);
+ db.commit();
+ when(fileStatuses.isDataUnchanged(any(Component.class))).thenReturn(true);
+ // this new value won't be persisted
+ measureRepository.addRawMeasure(REF_4, BUGS.getKey(), newMeasureBuilder().create(oldMeasure.getValue() + 1, 0));
+ step().execute(context);
+ assertThat(selectMeasure("file-uuid", BUGS).get().getValue()).isEqualTo(oldMeasure.getValue());
+ }
+
+ @Test
+ public void dont_keep_measures_for_unchanged_files() {
+ prepareProject();
+ LiveMeasureDto oldMeasure = insertMeasure("file-uuid", "project-uuid", BUGS);
+ db.commit();
+ when(fileStatuses.isDataUnchanged(any(Component.class))).thenReturn(false);
+ // this new value will be persisted
+ measureRepository.addRawMeasure(REF_4, BUGS.getKey(), newMeasureBuilder().create(oldMeasure.getValue() + 1, 0));
+ step().execute(context);
+ assertThat(selectMeasure("file-uuid", BUGS).get().getValue()).isEqualTo(oldMeasure.getValue() + 1);
+ }
+
+ @Test
public void persist_live_measures_of_portfolio_analysis() {
preparePortfolio();
@@ -190,7 +218,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
measureRepository.addRawMeasure(REF_2, STRING_METRIC.getKey(), newMeasureBuilder().create("subview-value"));
measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("project-value"));
- TestComputationStepContext context = new TestComputationStepContext();
step().execute(context);
assertThat(db.countRowsOfTable("live_measures")).isEqualTo(3);
@@ -288,7 +315,8 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest {
@Override
protected ComputationStep step() {
- return new PersistLiveMeasuresStep(dbClient, metricRepository, new MeasureToMeasureDto(analysisMetadataHolder, treeRootHolder), treeRootHolder, measureRepository);
+ return new PersistLiveMeasuresStep(dbClient, metricRepository, new MeasureToMeasureDto(analysisMetadataHolder, treeRootHolder), treeRootHolder, measureRepository,
+ Optional.of(fileStatuses));
}
private static void verifyStatistics(TestComputationStepContext context, int expectedInsertsOrUpdates) {
diff --git a/server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/component/ReportComponent.java b/server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/component/ReportComponent.java
index 575bcfeb006..5bcbcd9d11c 100644
--- a/server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/component/ReportComponent.java
+++ b/server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/component/ReportComponent.java
@@ -36,7 +36,7 @@ import static java.util.Objects.requireNonNull;
*/
public class ReportComponent implements Component {
- private static final FileAttributes DEFAULT_FILE_ATTRIBUTES = new FileAttributes(false, null, 1);
+ private static final FileAttributes DEFAULT_FILE_ATTRIBUTES = new FileAttributes(false, null, 1, false);
public static final Component DUMB_PROJECT = builder(Type.PROJECT, 1)
.setKey("PROJECT_KEY")
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileHashesDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileHashesDto.java
new file mode 100644
index 00000000000..456f1979b5b
--- /dev/null
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileHashesDto.java
@@ -0,0 +1,92 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.db.source;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+
+public class FileHashesDto {
+ protected String uuid;
+ protected String fileUuid;
+ protected String srcHash;
+ protected String dataHash;
+ protected String revision;
+ protected long updatedAt;
+ @Nullable
+ protected Integer lineHashesVersion;
+
+ public int getLineHashesVersion() {
+ return lineHashesVersion != null ? lineHashesVersion : LineHashVersion.WITHOUT_SIGNIFICANT_CODE.getDbValue();
+ }
+
+ public String getUuid() {
+ return uuid;
+ }
+
+ public String getFileUuid() {
+ return fileUuid;
+ }
+
+ public FileHashesDto setFileUuid(String fileUuid) {
+ this.fileUuid = fileUuid;
+ return this;
+ }
+ @CheckForNull
+ public String getDataHash() {
+ return dataHash;
+ }
+
+
+
+ /**
+ * MD5 of column BINARY_DATA. Used to know to detect data changes and need for update.
+ */
+ public FileHashesDto setDataHash(String s) {
+ this.dataHash = s;
+ return this;
+ }
+
+ @CheckForNull
+ public String getSrcHash() {
+ return srcHash;
+ }
+
+ /**
+ * Hash of file content. Value is computed by batch.
+ */
+ public FileHashesDto setSrcHash(@Nullable String srcHash) {
+ this.srcHash = srcHash;
+ return this;
+ }
+
+ public String getRevision() {
+ return revision;
+ }
+
+ public FileHashesDto setRevision(@Nullable String revision) {
+ this.revision = revision;
+ return this;
+ }
+
+ public long getUpdatedAt() {
+ return updatedAt;
+ }
+
+}
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java
index a2c4324e266..42c276ba030 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java
@@ -50,6 +50,13 @@ public class FileSourceDao implements Dao {
return version == null ? null : LineHashVersion.valueOf(version);
}
+ /**
+ * The returning object doesn't contain all fields filled. For example, binary data is not loaded.
+ */
+ public void scrollFileHashesByProjectUuid(DbSession dbSession, String projectUuid, ResultHandler<FileHashesDto> rowHandler) {
+ mapper(dbSession).scrollHashesForProject(projectUuid, rowHandler);
+ }
+
@CheckForNull
public List<String> selectLineHashes(DbSession dbSession, String fileUuid) {
Connection connection = dbSession.getConnection();
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDto.java
index 7b92a27d47e..e4525cec791 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDto.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDto.java
@@ -28,7 +28,6 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
-import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import net.jpountz.lz4.LZ4BlockInputStream;
import net.jpountz.lz4.LZ4BlockOutputStream;
@@ -38,7 +37,7 @@ import org.sonar.db.protobuf.DbFileSources;
import static com.google.common.base.Splitter.on;
import static java.lang.String.format;
-public class FileSourceDto {
+public class FileSourceDto extends FileHashesDto {
private static final String SIZE_LIMIT_EXCEEDED_EXCEPTION_MESSAGE = "Protocol message was too large. May be malicious. " +
"Use CodedInputStream.setSizeLimit() to increase the size limit.";
@@ -46,11 +45,8 @@ public class FileSourceDto {
public static final Splitter LINES_HASHES_SPLITTER = on('\n');
public static final int LINE_COUNT_NOT_POPULATED = -1;
- private String uuid;
private String projectUuid;
- private String fileUuid;
private long createdAt;
- private long updatedAt;
private String lineHashes;
/**
* When {@code line_count} column has been added, it's been populated with value {@link #LINE_COUNT_NOT_POPULATED -1},
@@ -64,26 +60,13 @@ public class FileSourceDto {
* {@code line_hashes}.
*/
private int lineCount = LINE_COUNT_NOT_POPULATED;
- private String srcHash;
private byte[] binaryData = new byte[0];
- private String dataHash;
- private String revision;
- @Nullable
- private Integer lineHashesVersion;
-
- public int getLineHashesVersion() {
- return lineHashesVersion != null ? lineHashesVersion : LineHashVersion.WITHOUT_SIGNIFICANT_CODE.getDbValue();
- }
public FileSourceDto setLineHashesVersion(int lineHashesVersion) {
this.lineHashesVersion = lineHashesVersion;
return this;
}
- public String getUuid() {
- return uuid;
- }
-
public FileSourceDto setUuid(String uuid) {
this.uuid = uuid;
return this;
@@ -98,20 +81,11 @@ public class FileSourceDto {
return this;
}
- public String getFileUuid() {
- return fileUuid;
- }
-
public FileSourceDto setFileUuid(String fileUuid) {
this.fileUuid = fileUuid;
return this;
}
- @CheckForNull
- public String getDataHash() {
- return dataHash;
- }
-
/**
* MD5 of column BINARY_DATA. Used to know to detect data changes and need for update.
*/
@@ -234,11 +208,6 @@ public class FileSourceDto {
return this;
}
- @CheckForNull
- public String getSrcHash() {
- return srcHash;
- }
-
/**
* Hash of file content. Value is computed by batch.
*/
@@ -256,19 +225,11 @@ public class FileSourceDto {
return this;
}
- public long getUpdatedAt() {
- return updatedAt;
- }
-
public FileSourceDto setUpdatedAt(long updatedAt) {
this.updatedAt = updatedAt;
return this;
}
- public String getRevision() {
- return revision;
- }
-
public FileSourceDto setRevision(@Nullable String revision) {
this.revision = revision;
return this;
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java
index 68580975c11..a77fb535216 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java
@@ -20,14 +20,13 @@
package org.sonar.db.source;
import java.util.Collection;
-import java.util.List;
import javax.annotation.CheckForNull;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.session.ResultHandler;
public interface FileSourceMapper {
- List<FileSourceDto> selectHashesForProject(@Param("projectUuid") String projectUuid);
+ void scrollHashesForProject(@Param("projectUuid") String projectUuid, ResultHandler<FileHashesDto> rowHandler);
@CheckForNull
FileSourceDto selectByFileUuid(@Param("fileUuid") String fileUuid);
diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml
index 608faddf398..53bba0f8365 100644
--- a/server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml
+++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml
@@ -24,14 +24,15 @@
file_uuid = #{fileUuid,jdbcType=VARCHAR}
</select>
- <select id="selectHashesForProject" parameterType="map" resultType="org.sonar.db.source.FileSourceDto">
+ <select id="scrollHashesForProject" parameterType="map" resultType="org.sonar.db.source.FileHashesDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
select
uuid,
file_uuid as fileUuid,
data_hash as dataHash,
src_hash as srcHash,
revision,
- updated_at as updatedAt
+ updated_at as updatedAt,
+ line_hashes_version as lineHashesVersion
from
file_sources
where
diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java
index aabd15fe3f4..885ed6d3bde 100644
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java
@@ -22,7 +22,9 @@ package org.sonar.db.source;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.stream.Collectors;
@@ -232,6 +234,43 @@ public class FileSourceDaoTest {
}
@Test
+ public void scrollFileHashes_handles_scrolling_more_than_1000_files() {
+ ComponentDto project = dbTester.components().insertPrivateProject();
+ List<ComponentDto> files = IntStream.range(0, 1001 + new Random().nextInt(5))
+ .mapToObj(i -> {
+ ComponentDto file = dbTester.components().insertComponent(newFileDto(project));
+ dbTester.fileSources().insertFileSource(file);
+ return file;
+ })
+ .collect(Collectors.toList());
+
+ Map<String, FileHashesDto> fileSourcesByUuid = new HashMap<>();
+ underTest.scrollFileHashesByProjectUuid(dbSession, project.projectUuid(), result -> fileSourcesByUuid.put(result.getResultObject().getFileUuid(), result.getResultObject()));
+
+ assertThat(fileSourcesByUuid).hasSize(files.size());
+ files.forEach(t -> assertThat(fileSourcesByUuid).containsKey(t.uuid()));
+ }
+
+ @Test
+ public void scrollFileHashes_returns_all_hashes() {
+ ComponentDto project = dbTester.components().insertPrivateProject();
+ ComponentDto file = dbTester.components().insertComponent(newFileDto(project));
+ FileSourceDto inserted = dbTester.fileSources().insertFileSource(file);
+
+ List<FileHashesDto> fileSources = new ArrayList<>(1);
+ underTest.scrollFileHashesByProjectUuid(dbSession, project.projectUuid(), result -> fileSources.add(result.getResultObject()));
+
+ assertThat(fileSources).hasSize(1);
+ FileHashesDto fileSource = fileSources.iterator().next();
+
+ assertThat(fileSource.getDataHash()).isEqualTo(inserted.getDataHash());
+ assertThat(fileSource.getFileUuid()).isEqualTo(inserted.getFileUuid());
+ assertThat(fileSource.getRevision()).isEqualTo(inserted.getRevision());
+ assertThat(fileSource.getSrcHash()).isEqualTo(inserted.getSrcHash());
+ assertThat(fileSource.getLineHashesVersion()).isEqualTo(inserted.getLineHashesVersion());
+ }
+
+ @Test
public void scrollLineHashes_handles_scrolling_more_than_1000_files() {
ComponentDto project = new Random().nextBoolean() ? dbTester.components().insertPrivateProject() : dbTester.components().insertPublicProject();
List<ComponentDto> files = IntStream.range(0, 1001 + new Random().nextInt(5))
diff --git a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputComponent.java b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputComponent.java
index cb37d133823..431575f1b54 100644
--- a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputComponent.java
+++ b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputComponent.java
@@ -52,6 +52,7 @@ public abstract class DefaultInputComponent implements InputComponent {
return id;
}
+
@Override
public int hashCode() {
return key().hashCode();
@@ -68,5 +69,6 @@ public abstract class DefaultInputComponent implements InputComponent {
public boolean hasMeasureFor(Metric metric) {
return storedMetricKeys.contains(metric.key());
+
}
}
diff --git a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java
index 888763dfdb7..382461fbc5a 100644
--- a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java
+++ b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java
@@ -70,6 +70,8 @@ public class DefaultInputFile extends DefaultInputComponent implements InputFile
private Metadata metadata;
private Collection<int[]> ignoreIssuesOnlineRanges;
private BitSet executableLines;
+ private boolean markedAsUnchanged;
+
public DefaultInputFile(DefaultIndexedFile indexedFile, Consumer<DefaultInputFile> metadataGenerator) {
this(indexedFile, metadataGenerator, null);
@@ -81,6 +83,7 @@ public class DefaultInputFile extends DefaultInputComponent implements InputFile
this.indexedFile = indexedFile;
this.metadataGenerator = metadataGenerator;
this.metadata = null;
+ this.markedAsUnchanged = false;
this.published = false;
this.excludedForCoverage = false;
this.contents = contents;
@@ -99,6 +102,15 @@ public class DefaultInputFile extends DefaultInputComponent implements InputFile
ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE);
}
+ public boolean isMarkedAsUnchanged() {
+ return markedAsUnchanged;
+ }
+
+ public DefaultInputComponent setMarkedAsUnchanged(boolean markedAsUnchanged) {
+ this.markedAsUnchanged = markedAsUnchanged;
+ return this;
+ }
+
@Override
public String contents() throws IOException {
if (contents != null) {
diff --git a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
index 98b48f12382..5c03698dc68 100644
--- a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
+++ b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
@@ -414,6 +414,11 @@ public class SensorContextTester implements SensorContext {
}
@Override
+ public void markAsUnchanged(InputFile inputFile) {
+ ((DefaultInputFile) inputFile).setMarkedAsUnchanged(true);
+ }
+
+ @Override
public WriteCache nextCache() {
return writeCache;
}
@@ -427,7 +432,6 @@ public class SensorContextTester implements SensorContext {
return readCache;
}
-
public void setPreviousCache(ReadCache cache) {
this.readCache = cache;
}
@@ -437,7 +441,6 @@ public class SensorContextTester implements SensorContext {
return cacheEnabled;
}
-
public void setCacheEnabled(boolean enabled) {
this.cacheEnabled = enabled;
}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java
index 9fe8fc5a7f5..4ca7c8be780 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java
@@ -66,6 +66,7 @@ public class ComponentsPublisher implements ReportPublisherStep {
fileBuilder.setIsTest(file.type() == InputFile.Type.TEST);
fileBuilder.setLines(file.lines());
fileBuilder.setStatus(convert(file.status()));
+ fileBuilder.setMarkedAsUnchanged(file.isMarkedAsUnchanged());
String lang = getLanguageKey(file);
if (lang != null) {
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java
index 7aaf4b5eebe..f2c25f372e4 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java
@@ -186,6 +186,10 @@ public class DefaultSensorStorage implements SensorStorage {
reportPublisher.getWriter().appendComponentMeasure(((DefaultInputComponent) component).scannerId(), toReportMeasure(measure));
}
+ public boolean hasIssues(DefaultInputComponent inputComponent) {
+ return reportPublisher.getReader().hasIssues(inputComponent.scannerId());
+ }
+
public static ScannerReport.Measure toReportMeasure(DefaultMeasure measureToSave) {
ScannerReport.Measure.Builder builder = ScannerReport.Measure.newBuilder();
builder.setMetricKey(measureToSave.metric().key());
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java
index 86c54c41d69..a573f94ff0e 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java
@@ -27,7 +27,6 @@ import org.sonar.api.batch.fs.internal.DefaultInputProject;
import org.sonar.api.batch.rule.ActiveRules;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.sonar.api.batch.sensor.cache.WriteCache;
-import org.sonar.api.batch.sensor.internal.SensorStorage;
import org.sonar.api.config.Configuration;
import org.sonar.api.config.Settings;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
@@ -39,7 +38,7 @@ public class ModuleSensorContext extends ProjectSensorContext {
private final InputModule module;
public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, Settings mutableModuleSettings, FileSystem fs, ActiveRules activeRules,
- SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration,
+ DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration,
WriteCache writeCache, ReadCache readCache, AnalysisCacheEnabled analysisCacheEnabled) {
super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled);
this.module = module;
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java
index a4cd85157d6..4c72730b94c 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java
@@ -40,7 +40,6 @@ import org.sonar.api.batch.sensor.cpd.internal.DefaultCpdTokens;
import org.sonar.api.batch.sensor.error.NewAnalysisError;
import org.sonar.api.batch.sensor.highlighting.NewHighlighting;
import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting;
-import org.sonar.api.batch.sensor.internal.SensorStorage;
import org.sonar.api.batch.sensor.issue.NewExternalIssue;
import org.sonar.api.batch.sensor.issue.NewIssue;
import org.sonar.api.batch.sensor.issue.internal.DefaultExternalIssue;
@@ -67,7 +66,7 @@ public class ProjectSensorContext implements SensorContext {
private final Settings mutableSettings;
private final FileSystem fs;
private final ActiveRules activeRules;
- private final SensorStorage sensorStorage;
+ private final DefaultSensorStorage sensorStorage;
private final DefaultInputProject project;
private final SonarRuntime sonarRuntime;
private final Configuration config;
@@ -77,7 +76,7 @@ public class ProjectSensorContext implements SensorContext {
private final AnalysisCacheEnabled analysisCacheEnabled;
public ProjectSensorContext(DefaultInputProject project, Configuration config, Settings mutableSettings, FileSystem fs, ActiveRules activeRules,
- SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache,
+ DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache,
AnalysisCacheEnabled analysisCacheEnabled) {
this.project = project;
this.config = config;
@@ -194,6 +193,12 @@ public class ProjectSensorContext implements SensorContext {
}
@Override
+ public void markAsUnchanged(InputFile inputFile) {
+ DefaultInputFile defaultInputFile = (DefaultInputFile) inputFile;
+ defaultInputFile.setMarkedAsUnchanged(true);
+ }
+
+ @Override
public WriteCache nextCache() {
return writeCache;
}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java
index cc440601a2f..ae2f877d384 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java
@@ -24,6 +24,7 @@ import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
+import java.util.List;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -40,6 +41,7 @@ import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.scanner.bootstrap.DefaultScannerWsClient;
import org.sonar.scanner.bootstrap.GlobalAnalysisMode;
import org.sonar.scanner.fs.InputModuleHierarchy;
+import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.scan.ScanProperties;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonarqube.ws.Ce;
@@ -93,6 +95,14 @@ public class ReportPublisherTest {
}
@Test
+ public void checks_if_component_has_issues() {
+ underTest.getWriter().writeComponentIssues(1, List.of(ScannerReport.Issue.newBuilder().build()));
+
+ assertThat(underTest.getReader().hasIssues(1)).isTrue();
+ assertThat(underTest.getReader().hasIssues(2)).isFalse();
+ }
+
+ @Test
public void use_30s_write_timeout() {
MockWsResponse submitMockResponse = new MockWsResponse();
submitMockResponse.setContent(Ce.SubmitResponse.newBuilder().setTaskId("task-1234").build().toByteArray());
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java
index bb5d8f78bea..877e2e3e8ac 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java
@@ -205,6 +205,16 @@ public class DefaultSensorStorageTest {
}
@Test
+ public void has_issues_delegates_to_report_publisher() {
+ DefaultInputFile file1 = new TestInputFileBuilder("foo", "src/Foo1.php").setStatus(InputFile.Status.SAME).build();
+ DefaultInputFile file2 = new TestInputFileBuilder("foo", "src/Foo2.php").setStatus(InputFile.Status.SAME).build();
+
+ reportWriter.writeComponentIssues(file1.scannerId(), List.of(ScannerReport.Issue.newBuilder().build()));
+ assertThat(underTest.hasIssues(file1)).isTrue();
+ assertThat(underTest.hasIssues(file2)).isFalse();
+ }
+
+ @Test
public void should_save_highlighting() {
DefaultInputFile file = new TestInputFileBuilder("foo", "src/Foo.php")
.setContents("// comment").build();
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java
index d40c7344bee..fb719656262 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java
@@ -32,7 +32,6 @@ import org.sonar.api.batch.fs.internal.DefaultInputProject;
import org.sonar.api.batch.measure.MetricFinder;
import org.sonar.api.batch.rule.ActiveRules;
import org.sonar.api.batch.rule.internal.ActiveRulesBuilder;
-import org.sonar.api.batch.sensor.internal.SensorStorage;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.internal.SonarRuntimeImpl;
import org.sonar.api.measures.CoreMetrics;
@@ -55,7 +54,7 @@ public class ModuleSensorContextTest {
private DefaultFileSystem fs;
private ModuleSensorContext adaptor;
private MapSettings settings;
- private SensorStorage sensorStorage;
+ private DefaultSensorStorage sensorStorage;
private SonarRuntime runtime;
private BranchConfiguration branchConfiguration;
private WriteCacheImpl writeCache;
@@ -70,7 +69,7 @@ public class ModuleSensorContextTest {
when(metricFinder.<Integer>findByKey(CoreMetrics.NCLOC_KEY)).thenReturn(CoreMetrics.NCLOC);
when(metricFinder.<String>findByKey(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY)).thenReturn(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION);
settings = new MapSettings();
- sensorStorage = mock(SensorStorage.class);
+ sensorStorage = mock(DefaultSensorStorage.class);
branchConfiguration = mock(BranchConfiguration.class);
writeCache = mock(WriteCacheImpl.class);
readCache = mock(ReadCacheImpl.class);
diff --git a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java
index 006bdf66760..b4ba603055e 100644
--- a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java
+++ b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java
@@ -108,6 +108,11 @@ public class ScannerReportReader {
return emptyCloseableIterator();
}
+ public boolean hasIssues(int componentRef) {
+ File file = fileStructure.fileFor(FileStructure.Domain.ISSUES, componentRef);
+ return fileExists(file);
+ }
+
public CloseableIterator<ScannerReport.ExternalIssue> readComponentExternalIssues(int componentRef) {
File file = fileStructure.fileFor(FileStructure.Domain.EXTERNAL_ISSUES, componentRef);
if (fileExists(file)) {
diff --git a/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto b/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto
index 9b2b159dc31..4870c034282 100644
--- a/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto
+++ b/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto
@@ -131,6 +131,7 @@ message Component {
// Path relative to project base directory
string project_relative_path = 14;
+ bool markedAsUnchanged = 15;
enum ComponentType {
UNSET = 0;