diff options
author | Duarte Meneses <duarte.meneses@sonarsource.com> | 2022-06-22 16:17:26 -0500 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2022-07-23 20:02:53 +0000 |
commit | 0fb5e45d935ad212aa3fe32202c33dd8395078c5 (patch) | |
tree | 4c2afc044dc034f9c70a11b58f02a762b3ba3448 | |
parent | 1220331cf405fb916a005284120b0ed02ea67ac2 (diff) | |
download | sonarqube-0fb5e45d935ad212aa3fe32202c33dd8395078c5.tar.gz sonarqube-0fb5e45d935ad212aa3fe32202c33dd8395078c5.zip |
SONAR-17044 Optimize Compute Engine issue tracking and persisting of measures when file is marked as unchanged
46 files changed, 1296 insertions, 174 deletions
diff --git a/build.gradle b/build.gradle index 9339b64e526..32fa4bc083e 100644 --- a/build.gradle +++ b/build.gradle @@ -178,7 +178,7 @@ subprojects { dependency 'org.sonarsource.kotlin:sonar-kotlin-plugin:2.9.0.1147' dependency 'org.sonarsource.slang:sonar-ruby-plugin:1.9.0.3429' dependency 'org.sonarsource.slang:sonar-scala-plugin:1.9.0.3429' - dependency 'org.sonarsource.api.plugin:sonar-plugin-api:9.8.0.203' + dependency 'org.sonarsource.api.plugin:sonar-plugin-api:9.9.0.228' dependency 'org.sonarsource.xml:sonar-xml-plugin:2.5.0.3376' dependency 'org.sonarsource.iac:sonar-iac-plugin:1.7.0.2012' dependency 'org.sonarsource.text:sonar-text-plugin:1.1.0.282' diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java index 9f000f49013..ef3d0cce1e5 100644 --- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java +++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java @@ -46,6 +46,8 @@ import org.sonar.xoo.rule.HasTagSensor; import org.sonar.xoo.rule.hotspot.HotspotWithSingleContextSensor; import org.sonar.xoo.rule.hotspot.HotspotWithoutContextSensor; import org.sonar.xoo.rule.hotspot.HotspotWithContextsSensor; +import org.sonar.xoo.rule.HotspotSensor; +import org.sonar.xoo.rule.MarkAsUnchangedSensor; import org.sonar.xoo.rule.MultilineIssuesSensor; import org.sonar.xoo.rule.NoSonarSensor; import org.sonar.xoo.rule.OneBlockerIssuePerFileSensor; @@ -183,6 +185,7 @@ public class XooPlugin implements Plugin { AnalysisErrorSensor.class, // Other + MarkAsUnchangedSensor.class, XooProjectBuilder.class, XooPostJob.class, XooIssueFilter.class, diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/MarkAsUnchangedSensor.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/MarkAsUnchangedSensor.java new file mode 100644 index 00000000000..68ece9802a8 --- /dev/null +++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/MarkAsUnchangedSensor.java @@ -0,0 +1,50 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.xoo.rule; + +import org.sonar.api.batch.fs.FilePredicates; +import org.sonar.api.batch.fs.FileSystem; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.sensor.Sensor; +import org.sonar.api.batch.sensor.SensorContext; +import org.sonar.api.batch.sensor.SensorDescriptor; +import org.sonar.xoo.Xoo; +import org.sonar.xoo.Xoo2; + +public class MarkAsUnchangedSensor implements Sensor { + public static final String ACTIVATE_MARK_AS_UNCHANGED = "sonar.markAsUnchanged"; + + @Override + public void describe(SensorDescriptor descriptor) { + descriptor.name("Mark As Unchanged Sensor") + .onlyOnLanguages(Xoo.KEY, Xoo2.KEY) + .onlyWhenConfiguration(c -> c.getBoolean(ACTIVATE_MARK_AS_UNCHANGED).orElse(false)) + .processesFilesIndependently(); + } + + @Override + public void execute(SensorContext context) { + FileSystem fs = context.fileSystem(); + FilePredicates p = fs.predicates(); + for (InputFile f : fs.inputFiles(p.all())) { + context.markAsUnchanged(f); + } + } +} diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/OneIssuePerLineSensor.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/OneIssuePerLineSensor.java index 74c713014a4..34d402e3672 100644 --- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/OneIssuePerLineSensor.java +++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/OneIssuePerLineSensor.java @@ -45,6 +45,7 @@ public class OneIssuePerLineSensor implements Sensor { descriptor .name("One Issue Per Line") .onlyOnLanguages(Xoo.KEY, Xoo2.KEY) + .onlyWhenConfiguration(c -> !c.getBoolean("sonar.markAsUnchanged").orElse(false)) .createIssuesForRuleRepositories(XooRulesDefinition.XOO_REPOSITORY, XooRulesDefinition.XOO2_REPOSITORY) .processesFilesIndependently(); } diff --git a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/MarkAsUnchangedSensorTest.java b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/MarkAsUnchangedSensorTest.java new file mode 100644 index 00000000000..2b9af3daee3 --- /dev/null +++ b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/MarkAsUnchangedSensorTest.java @@ -0,0 +1,74 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.xoo.rule; + +import java.io.IOException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.internal.DefaultInputFile; +import org.sonar.api.batch.fs.internal.TestInputFileBuilder; +import org.sonar.api.batch.sensor.internal.DefaultSensorDescriptor; +import org.sonar.api.batch.sensor.internal.SensorContextTester; +import org.sonar.api.config.Configuration; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.xoo.Xoo; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MarkAsUnchangedSensorTest { + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + private final MarkAsUnchangedSensor sensor = new MarkAsUnchangedSensor(); + + @Test + public void mark_as_unchanged_for_all_files() throws IOException { + SensorContextTester context = SensorContextTester.create(temp.newFolder()); + DefaultInputFile inputFile1 = createFile("file1"); + DefaultInputFile inputFile2 = createFile("file2"); + + context.fileSystem() + .add(inputFile1) + .add(inputFile2); + + sensor.execute(context); + assertThat(inputFile1.isMarkedAsUnchanged()).isTrue(); + assertThat(inputFile2.isMarkedAsUnchanged()).isTrue(); + } + + @Test + public void only_runs_if_property_is_set() { + DefaultSensorDescriptor descriptor = new DefaultSensorDescriptor(); + sensor.describe(descriptor); + Configuration configWithProperty = new MapSettings().setProperty("sonar.markAsUnchanged", "true").asConfig(); + Configuration configWithoutProperty = new MapSettings().asConfig(); + + assertThat(descriptor.configurationPredicate().test(configWithoutProperty)).isFalse(); + assertThat(descriptor.configurationPredicate().test(configWithProperty)).isTrue(); + } + + private DefaultInputFile createFile(String name) { + return new TestInputFileBuilder("foo", "src/" + name) + .setLanguage(Xoo.KEY) + .initMetadata("a\nb\nc\nd\ne\nf\ng\nh\ni\n") + .build(); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java index 1c6bf0aff1a..6808c7b6db8 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java @@ -345,7 +345,8 @@ public class ComponentTreeBuilder { return new FileAttributes( component.getIsTest(), lang != null ? lang.intern() : null, - component.getLines()); + component.getLines(), + component.getMarkedAsUnchanged()); } private static class Node { diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java index b523627c0dd..c1c868ec1f9 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java @@ -33,15 +33,25 @@ public class FileAttributes { private final boolean unitTest; @CheckForNull private final String languageKey; + private final boolean markedAsUnchanged; private final int lines; public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines) { + this(unitTest, languageKey, lines, false); + } + + public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines, boolean markedAsUnchanged) { this.unitTest = unitTest; this.languageKey = languageKey; + this.markedAsUnchanged = markedAsUnchanged; checkArgument(lines > 0, "Number of lines must be greater than zero"); this.lines = lines; } + public boolean isMarkedAsUnchanged() { + return markedAsUnchanged; + } + public boolean isUnitTest() { return unitTest; } @@ -64,6 +74,7 @@ public class FileAttributes { "languageKey='" + languageKey + '\'' + ", unitTest=" + unitTest + ", lines=" + lines + + ", markedAsUnchanged=" + markedAsUnchanged + '}'; } } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java new file mode 100644 index 00000000000..ab962b6dfc8 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java @@ -0,0 +1,30 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.component; + +public interface FileStatuses { + /** + * A file is unchanged compared to the last analysis if it was detected as unchanged by the scanner and + * it's confirmed to be unchanged by the CE, by comparing file hashes. + */ + boolean isUnchanged(Component component); + + boolean isDataUnchanged(Component component); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java new file mode 100644 index 00000000000..045b7ffa14e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java @@ -0,0 +1,98 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.component; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; +import org.sonar.ce.task.projectanalysis.source.SourceHashRepository; +import org.sonar.db.source.FileHashesDto; + +import static com.google.common.base.Preconditions.checkState; +import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER; + +public class FileStatusesImpl implements FileStatuses { + private final PreviousSourceHashRepository previousSourceHashRepository; + private final SourceHashRepository sourceHashRepository; + private final AnalysisMetadataHolder analysisMetadataHolder; + private final TreeRootHolder treeRootHolder; + private Set<String> fileUuidsMarkedAsUnchanged; + + public FileStatusesImpl(AnalysisMetadataHolder analysisMetadataHolder, TreeRootHolder treeRootHolder, PreviousSourceHashRepository previousSourceHashRepository, + SourceHashRepository sourceHashRepository) { + this.analysisMetadataHolder = analysisMetadataHolder; + this.treeRootHolder = treeRootHolder; + this.previousSourceHashRepository = previousSourceHashRepository; + this.sourceHashRepository = sourceHashRepository; + } + + public void initialize() { + fileUuidsMarkedAsUnchanged = new HashSet<>(); + if (!analysisMetadataHolder.isPullRequest() && !analysisMetadataHolder.isFirstAnalysis()) { + new DepthTraversalTypeAwareCrawler(new Visitor()).visit(treeRootHolder.getRoot()); + } + } + + private class Visitor extends TypeAwareVisitorAdapter { + private boolean canTrustUnchangedFlags = true; + + private Visitor() { + super(CrawlerDepthLimit.FILE, PRE_ORDER); + } + + @Override + public void visitFile(Component file) { + if (file.getStatus() != Component.Status.SAME || !canTrustUnchangedFlags) { + return; + } + + canTrustUnchangedFlags = hashEquals(file); + if (canTrustUnchangedFlags) { + if (file.getFileAttributes().isMarkedAsUnchanged()) { + fileUuidsMarkedAsUnchanged.add(file.getUuid()); + } + } else { + fileUuidsMarkedAsUnchanged.clear(); + } + } + } + + @Override + public boolean isUnchanged(Component component) { + failIfNotInitialized(); + return component.getStatus() == Component.Status.SAME && hashEquals(component); + } + + @Override + public boolean isDataUnchanged(Component component) { + failIfNotInitialized(); + return fileUuidsMarkedAsUnchanged.contains(component.getUuid()); + } + + private boolean hashEquals(Component component) { + Optional<String> dbHash = previousSourceHashRepository.getDbFile(component).map(FileHashesDto::getSrcHash); + return dbHash.map(hash -> hash.equals(sourceHashRepository.getRawSourceHash(component))).orElse(false); + } + + private void failIfNotInitialized() { + checkState(fileUuidsMarkedAsUnchanged != null, "Not initialized"); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepository.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepository.java new file mode 100644 index 00000000000..1eb4b72c5d8 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepository.java @@ -0,0 +1,27 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.component; + +import java.util.Optional; +import org.sonar.db.source.FileHashesDto; + +public interface PreviousSourceHashRepository { + Optional<FileHashesDto> getDbFile(Component component); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImpl.java new file mode 100644 index 00000000000..e13d01293db --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImpl.java @@ -0,0 +1,48 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.component; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import org.sonar.db.source.FileHashesDto; + +import static org.sonar.api.utils.Preconditions.checkNotNull; +import static org.sonar.api.utils.Preconditions.checkState; + +public class PreviousSourceHashRepositoryImpl implements PreviousSourceHashRepository { + private Map<String, FileHashesDto> previousFileHashesByUuid = null; + + public void set(Map<String, FileHashesDto> previousFileHashesByUuid) { + checkState(this.previousFileHashesByUuid == null, "Repository already initialized"); + checkNotNull(previousFileHashesByUuid); + this.previousFileHashesByUuid = Collections.unmodifiableMap(previousFileHashesByUuid); + } + + public Map<String, FileHashesDto> getMap() { + return previousFileHashesByUuid; + } + + @Override + public Optional<FileHashesDto> getDbFile(Component component) { + checkState(previousFileHashesByUuid != null, "Repository not initialized"); + return Optional.ofNullable(previousFileHashesByUuid.get(component.getUuid())); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java index c8045d67c54..89f249a57e2 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java @@ -34,6 +34,8 @@ import org.sonar.ce.task.projectanalysis.component.BranchLoader; import org.sonar.ce.task.projectanalysis.component.BranchPersisterImpl; import org.sonar.ce.task.projectanalysis.component.ConfigurationRepositoryImpl; import org.sonar.ce.task.projectanalysis.component.DisabledComponentsHolderImpl; +import org.sonar.ce.task.projectanalysis.component.FileStatusesImpl; +import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepositoryImpl; import org.sonar.ce.task.projectanalysis.component.ProjectPersister; import org.sonar.ce.task.projectanalysis.component.ReferenceBranchComponentUuids; import org.sonar.ce.task.projectanalysis.component.ReportModulesPath; @@ -193,6 +195,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop new ComputationTempFolderProvider(), ReportModulesPath.class, + FileStatusesImpl.class, new MetricModule(), // holders @@ -213,6 +216,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop SiblingComponentsWithOpenIssues.class, // repositories + PreviousSourceHashRepositoryImpl.class, LanguageRepositoryImpl.class, MeasureRepositoryImpl.class, EventRepositoryImpl.class, diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitor.java index 5a322ab7ca1..8f4c2405d82 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitor.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitor.java @@ -19,11 +19,13 @@ */ package org.sonar.ce.task.projectanalysis.issue; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Stream; import org.sonar.ce.task.projectanalysis.component.Component; import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit; +import org.sonar.ce.task.projectanalysis.component.FileStatuses; import org.sonar.ce.task.projectanalysis.component.ReferenceBranchComponentUuids; import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter; import org.sonar.ce.task.projectanalysis.util.cache.DiskCache.CacheAppender; @@ -37,49 +39,68 @@ public class IntegrateIssuesVisitor extends TypeAwareVisitorAdapter { private final ProtoIssueCache protoIssueCache; private final TrackerRawInputFactory rawInputFactory; + private final TrackerBaseInputFactory baseInputFactory; private final IssueLifecycle issueLifecycle; private final IssueVisitors issueVisitors; private final IssueTrackingDelegator issueTracking; private final SiblingsIssueMerger issueStatusCopier; private final ReferenceBranchComponentUuids referenceBranchComponentUuids; private final PullRequestSourceBranchMerger pullRequestSourceBranchMerger; + private final FileStatuses fileStatuses; public IntegrateIssuesVisitor( ProtoIssueCache protoIssueCache, TrackerRawInputFactory rawInputFactory, + TrackerBaseInputFactory baseInputFactory, IssueLifecycle issueLifecycle, IssueVisitors issueVisitors, IssueTrackingDelegator issueTracking, SiblingsIssueMerger issueStatusCopier, ReferenceBranchComponentUuids referenceBranchComponentUuids, - PullRequestSourceBranchMerger pullRequestSourceBranchMerger) { + PullRequestSourceBranchMerger pullRequestSourceBranchMerger, + FileStatuses fileStatuses) { super(CrawlerDepthLimit.FILE, POST_ORDER); this.protoIssueCache = protoIssueCache; this.rawInputFactory = rawInputFactory; + this.baseInputFactory = baseInputFactory; this.issueLifecycle = issueLifecycle; this.issueVisitors = issueVisitors; this.issueTracking = issueTracking; this.issueStatusCopier = issueStatusCopier; this.referenceBranchComponentUuids = referenceBranchComponentUuids; this.pullRequestSourceBranchMerger = pullRequestSourceBranchMerger; + this.fileStatuses = fileStatuses; } @Override public void visitAny(Component component) { try (CacheAppender<DefaultIssue> cacheAppender = protoIssueCache.newAppender()) { issueVisitors.beforeComponent(component); - Input<DefaultIssue> rawInput = rawInputFactory.create(component); - TrackingResult tracking = issueTracking.track(component, rawInput); - fillNewOpenIssues(component, tracking.newIssues(), rawInput, cacheAppender); - fillExistingOpenIssues(component, tracking.issuesToMerge(), cacheAppender); - closeIssues(component, tracking.issuesToClose(), cacheAppender); - copyIssues(component, tracking.issuesToCopy(), cacheAppender); + + if (fileStatuses.isDataUnchanged(component)) { + // we assume there's a previous analysis of the same branch + Input<DefaultIssue> baseIssues = baseInputFactory.create(component); + useBaseIssues(component, baseIssues.getIssues(), cacheAppender); + } else { + Input<DefaultIssue> rawInput = rawInputFactory.create(component); + TrackingResult tracking = issueTracking.track(component, rawInput); + fillNewOpenIssues(component, tracking.newIssues(), rawInput, cacheAppender); + fillExistingOpenIssues(component, tracking.issuesToMerge(), cacheAppender); + closeIssues(component, tracking.issuesToClose(), cacheAppender); + copyIssues(component, tracking.issuesToCopy(), cacheAppender); + } issueVisitors.afterComponent(component); } catch (Exception e) { throw new IllegalStateException(String.format("Fail to process issues of component '%s'", component.getDbKey()), e); } } + private void useBaseIssues(Component component, Collection<DefaultIssue> dbIssues, CacheAppender<DefaultIssue> cacheAppender) { + for (DefaultIssue issue : dbIssues) { + process(component, issue, cacheAppender); + } + } + private void fillNewOpenIssues(Component component, Stream<DefaultIssue> newIssues, Input<DefaultIssue> rawInput, CacheAppender<DefaultIssue> cacheAppender) { List<DefaultIssue> newIssuesList = newIssues .peek(issueLifecycle::initNewOpenIssue) @@ -127,8 +148,7 @@ public class IntegrateIssuesVisitor extends TypeAwareVisitorAdapter { private void process(Component component, DefaultIssue issue, CacheAppender<DefaultIssue> cacheAppender) { issueLifecycle.doAutomaticTransition(issue); issueVisitors.onIssue(component, issue); - if (issue.isNew() || issue.isChanged() || issue.isCopied() || - issue.isNoLongerNewCodeReferenceIssue() || issue.isToBeMigratedAsNewCodeReferenceIssue()) { + if (issue.isNew() || issue.isChanged() || issue.isCopied() || issue.isNoLongerNewCodeReferenceIssue() || issue.isToBeMigratedAsNewCodeReferenceIssue()) { cacheAppender.append(issue); } } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImpl.java index 679fea498d4..609a3e942e9 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImpl.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImpl.java @@ -28,8 +28,7 @@ import org.sonar.api.utils.log.Loggers; import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; import org.sonar.ce.task.projectanalysis.batch.BatchReportReader; import org.sonar.ce.task.projectanalysis.component.Component; -import org.sonar.ce.task.projectanalysis.component.Component.Status; -import org.sonar.ce.task.projectanalysis.source.SourceHashRepository; +import org.sonar.ce.task.projectanalysis.component.FileStatuses; import org.sonar.ce.task.projectanalysis.source.SourceLinesDiff; import org.sonar.scanner.protocol.output.ScannerReport; @@ -44,15 +43,15 @@ public class ScmInfoRepositoryImpl implements ScmInfoRepository { private final ScmInfoDbLoader scmInfoDbLoader; private final AnalysisMetadataHolder analysisMetadata; private final SourceLinesDiff sourceLinesDiff; - private final SourceHashRepository sourceHashRepository; + private final FileStatuses fileStatuses; public ScmInfoRepositoryImpl(BatchReportReader scannerReportReader, AnalysisMetadataHolder analysisMetadata, ScmInfoDbLoader scmInfoDbLoader, - SourceLinesDiff sourceLinesDiff, SourceHashRepository sourceHashRepository) { + SourceLinesDiff sourceLinesDiff, FileStatuses fileStatuses) { this.scannerReportReader = scannerReportReader; this.analysisMetadata = analysisMetadata; this.scmInfoDbLoader = scmInfoDbLoader; this.sourceLinesDiff = sourceLinesDiff; - this.sourceHashRepository = sourceHashRepository; + this.fileStatuses = fileStatuses; } @Override @@ -118,9 +117,7 @@ public class ScmInfoRepositoryImpl implements ScmInfoRepository { } ScmInfo scmInfo = keepAuthorAndRevision ? dbInfoOpt.get() : removeAuthorAndRevision(dbInfoOpt.get()); - boolean fileUnchanged = file.getStatus() == Status.SAME && sourceHashRepository.getRawSourceHash(file).equals(dbInfoOpt.get().fileHash()); - - if (fileUnchanged) { + if (fileStatuses.isUnchanged(file)) { return Optional.of(scmInfo); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStep.java index 66f6798ba02..7748b6206f3 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStep.java @@ -19,10 +19,7 @@ */ package org.sonar.ce.task.projectanalysis.source; -import com.google.common.collect.ImmutableMap; -import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.commons.codec.digest.DigestUtils; @@ -31,6 +28,7 @@ import org.sonar.api.utils.System2; import org.sonar.ce.task.projectanalysis.component.Component; import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit; import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler; +import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepository; import org.sonar.ce.task.projectanalysis.component.TreeRootHolder; import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter; import org.sonar.ce.task.projectanalysis.scm.Changeset; @@ -39,6 +37,7 @@ import org.sonar.core.util.UuidFactory; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.protobuf.DbFileSources; +import org.sonar.db.source.FileHashesDto; import org.sonar.db.source.FileSourceDto; import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER; @@ -51,10 +50,11 @@ public class PersistFileSourcesStep implements ComputationStep { private final FileSourceDataComputer fileSourceDataComputer; private final FileSourceDataWarnings fileSourceDataWarnings; private final UuidFactory uuidFactory; + private final PreviousSourceHashRepository previousSourceHashRepository; public PersistFileSourcesStep(DbClient dbClient, System2 system2, TreeRootHolder treeRootHolder, SourceLinesHashRepository sourceLinesHash, FileSourceDataComputer fileSourceDataComputer, - FileSourceDataWarnings fileSourceDataWarnings, UuidFactory uuidFactory) { + FileSourceDataWarnings fileSourceDataWarnings, UuidFactory uuidFactory, PreviousSourceHashRepository previousSourceHashRepository) { this.dbClient = dbClient; this.system2 = system2; this.treeRootHolder = treeRootHolder; @@ -62,6 +62,7 @@ public class PersistFileSourcesStep implements ComputationStep { this.fileSourceDataComputer = fileSourceDataComputer; this.fileSourceDataWarnings = fileSourceDataWarnings; this.uuidFactory = uuidFactory; + this.previousSourceHashRepository = previousSourceHashRepository; } @Override @@ -77,8 +78,6 @@ public class PersistFileSourcesStep implements ComputationStep { private class FileSourceVisitor extends TypeAwareVisitorAdapter { private final DbSession session; - - private Map<String, FileSourceDto> previousFileSourcesByUuid = new HashMap<>(); private String projectUuid; private FileSourceVisitor(DbSession session) { @@ -89,11 +88,6 @@ public class PersistFileSourcesStep implements ComputationStep { @Override public void visitProject(Component project) { this.projectUuid = project.getUuid(); - session.select("org.sonar.db.source.FileSourceMapper.selectHashesForProject", ImmutableMap.of("projectUuid", projectUuid), - context -> { - FileSourceDto dto = (FileSourceDto) context.getResultObject(); - previousFileSourcesByUuid.put(dto.getFileUuid(), dto); - }); } @Override @@ -115,7 +109,7 @@ public class PersistFileSourcesStep implements ComputationStep { List<String> lineHashes = fileSourceData.getLineHashes(); Changeset latestChangeWithRevision = fileSourceData.getLatestChangeWithRevision(); int lineHashesVersion = sourceLinesHash.getLineHashesVersion(file); - FileSourceDto previousDto = previousFileSourcesByUuid.get(file.getUuid()); + FileHashesDto previousDto = previousSourceHashRepository.getDbFile(file).orElse(null); if (previousDto == null) { FileSourceDto dto = new FileSourceDto() .setUuid(uuidFactory.create()) @@ -139,7 +133,8 @@ public class PersistFileSourcesStep implements ComputationStep { boolean revisionUpdated = !ObjectUtils.equals(revision, previousDto.getRevision()); boolean lineHashesVersionUpdated = previousDto.getLineHashesVersion() != lineHashesVersion; if (binaryDataUpdated || srcHashUpdated || revisionUpdated || lineHashesVersionUpdated) { - previousDto + FileSourceDto updatedDto = new FileSourceDto() + .setUuid(previousDto.getUuid()) .setBinaryData(binaryData) .setDataHash(dataHash) .setSrcHash(srcHash) @@ -147,7 +142,7 @@ public class PersistFileSourcesStep implements ComputationStep { .setLineHashesVersion(lineHashesVersion) .setRevision(revision) .setUpdatedAt(system2.now()); - dbClient.fileSourceDao().update(session, previousDto); + dbClient.fileSourceDao().update(session, updatedDto); session.commit(); } } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStep.java new file mode 100644 index 00000000000..22e885088ad --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStep.java @@ -0,0 +1,68 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.util.HashMap; +import java.util.Map; +import org.sonar.ce.task.projectanalysis.component.FileStatusesImpl; +import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepositoryImpl; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolder; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.source.FileHashesDto; +import org.sonar.db.source.FileSourceDao; + +public class LoadFileHashesAndStatusStep implements ComputationStep { + private final DbClient dbClient; + private final PreviousSourceHashRepositoryImpl previousFileHashesRepository; + private final FileStatusesImpl fileStatuses; + private final FileSourceDao fileSourceDao; + private final TreeRootHolder treeRootHolder; + + public LoadFileHashesAndStatusStep(DbClient dbClient, PreviousSourceHashRepositoryImpl previousFileHashesRepository, + FileStatusesImpl fileStatuses, FileSourceDao fileSourceDao, TreeRootHolder treeRootHolder) { + this.dbClient = dbClient; + this.previousFileHashesRepository = previousFileHashesRepository; + this.fileStatuses = fileStatuses; + this.fileSourceDao = fileSourceDao; + this.treeRootHolder = treeRootHolder; + } + + @Override + public void execute(Context context) { + Map<String, FileHashesDto> previousFileHashesByUuid = new HashMap<>(); + String projectUuid = treeRootHolder.getRoot().getUuid(); + + try (DbSession session = dbClient.openSession(false)) { + fileSourceDao.scrollFileHashesByProjectUuid(session, projectUuid, ctx -> { + FileHashesDto dto = ctx.getResultObject(); + previousFileHashesByUuid.put(dto.getFileUuid(), dto); + }); + } + previousFileHashesRepository.set(previousFileHashesByUuid); + fileStatuses.initialize(); + } + + @Override + public String getDescription() { + return "Load file hashes and statuses"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java index f21a9e73a80..e9dd25a1df1 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStep.java @@ -22,12 +22,14 @@ package org.sonar.ce.task.projectanalysis.step; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import javax.annotation.Nonnull; import org.sonar.ce.task.projectanalysis.component.Component; import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit; import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler; +import org.sonar.ce.task.projectanalysis.component.FileStatuses; import org.sonar.ce.task.projectanalysis.component.TreeRootHolder; import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter; import org.sonar.ce.task.projectanalysis.measure.BestValueOptimization; @@ -41,8 +43,55 @@ import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.measure.LiveMeasureDto; +import static org.sonar.api.measures.CoreMetrics.BLOCKER_VIOLATIONS_KEY; +import static org.sonar.api.measures.CoreMetrics.BUGS_KEY; +import static org.sonar.api.measures.CoreMetrics.CLASSES_KEY; +import static org.sonar.api.measures.CoreMetrics.CLASS_COMPLEXITY_KEY; +import static org.sonar.api.measures.CoreMetrics.CODE_SMELLS_KEY; +import static org.sonar.api.measures.CoreMetrics.COGNITIVE_COMPLEXITY_KEY; +import static org.sonar.api.measures.CoreMetrics.COMMENT_LINES_DENSITY_KEY; +import static org.sonar.api.measures.CoreMetrics.COMMENT_LINES_KEY; +import static org.sonar.api.measures.CoreMetrics.COMPLEXITY_IN_CLASSES_KEY; +import static org.sonar.api.measures.CoreMetrics.COMPLEXITY_IN_FUNCTIONS_KEY; +import static org.sonar.api.measures.CoreMetrics.COMPLEXITY_KEY; +import static org.sonar.api.measures.CoreMetrics.CONFIRMED_ISSUES_KEY; +import static org.sonar.api.measures.CoreMetrics.CRITICAL_VIOLATIONS_KEY; +import static org.sonar.api.measures.CoreMetrics.DEVELOPMENT_COST_KEY; +import static org.sonar.api.measures.CoreMetrics.EFFORT_TO_REACH_MAINTAINABILITY_RATING_A_KEY; +import static org.sonar.api.measures.CoreMetrics.FALSE_POSITIVE_ISSUES_KEY; +import static org.sonar.api.measures.CoreMetrics.FILES_KEY; import static org.sonar.api.measures.CoreMetrics.FILE_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.FILE_COMPLEXITY_KEY; +import static org.sonar.api.measures.CoreMetrics.FUNCTIONS_KEY; import static org.sonar.api.measures.CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.FUNCTION_COMPLEXITY_KEY; +import static org.sonar.api.measures.CoreMetrics.GENERATED_LINES_KEY; +import static org.sonar.api.measures.CoreMetrics.GENERATED_NCLOC_KEY; +import static org.sonar.api.measures.CoreMetrics.INFO_VIOLATIONS_KEY; +import static org.sonar.api.measures.CoreMetrics.LINES_KEY; +import static org.sonar.api.measures.CoreMetrics.MAJOR_VIOLATIONS_KEY; +import static org.sonar.api.measures.CoreMetrics.MINOR_VIOLATIONS_KEY; +import static org.sonar.api.measures.CoreMetrics.NCLOC_DATA_KEY; +import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; +import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY; +import static org.sonar.api.measures.CoreMetrics.OPEN_ISSUES_KEY; +import static org.sonar.api.measures.CoreMetrics.RELIABILITY_RATING_KEY; +import static org.sonar.api.measures.CoreMetrics.RELIABILITY_REMEDIATION_EFFORT_KEY; +import static org.sonar.api.measures.CoreMetrics.REOPENED_ISSUES_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_REVIEWED_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_REVIEWED_STATUS_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_TO_REVIEW_STATUS_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_RATING_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_REMEDIATION_EFFORT_KEY; +import static org.sonar.api.measures.CoreMetrics.SECURITY_REVIEW_RATING_KEY; +import static org.sonar.api.measures.CoreMetrics.SQALE_DEBT_RATIO_KEY; +import static org.sonar.api.measures.CoreMetrics.SQALE_RATING_KEY; +import static org.sonar.api.measures.CoreMetrics.STATEMENTS_KEY; +import static org.sonar.api.measures.CoreMetrics.TECHNICAL_DEBT_KEY; +import static org.sonar.api.measures.CoreMetrics.VIOLATIONS_KEY; +import static org.sonar.api.measures.CoreMetrics.VULNERABILITIES_KEY; +import static org.sonar.api.measures.CoreMetrics.WONT_FIX_ISSUES_KEY; import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER; public class PersistLiveMeasuresStep implements ComputationStep { @@ -51,20 +100,31 @@ public class PersistLiveMeasuresStep implements ComputationStep { * List of metrics that should not be persisted on file measure. */ private static final Set<String> NOT_TO_PERSIST_ON_FILE_METRIC_KEYS = Set.of(FILE_COMPLEXITY_DISTRIBUTION_KEY, FUNCTION_COMPLEXITY_DISTRIBUTION_KEY); - + private static final Set<String> NOT_TO_PERSIST_ON_UNCHANGED_FILES = Set.of( + BLOCKER_VIOLATIONS_KEY, BUGS_KEY, CLASS_COMPLEXITY_KEY, CLASSES_KEY, CODE_SMELLS_KEY, COGNITIVE_COMPLEXITY_KEY, COMMENT_LINES_KEY, COMMENT_LINES_DENSITY_KEY, + COMPLEXITY_KEY, COMPLEXITY_IN_CLASSES_KEY, COMPLEXITY_IN_FUNCTIONS_KEY, CONFIRMED_ISSUES_KEY, CRITICAL_VIOLATIONS_KEY, DEVELOPMENT_COST_KEY, + EFFORT_TO_REACH_MAINTAINABILITY_RATING_A_KEY, FALSE_POSITIVE_ISSUES_KEY, FILE_COMPLEXITY_KEY, FILE_COMPLEXITY_DISTRIBUTION_KEY, FILES_KEY, FUNCTION_COMPLEXITY_KEY, + FUNCTION_COMPLEXITY_DISTRIBUTION_KEY, FUNCTIONS_KEY, GENERATED_LINES_KEY, GENERATED_NCLOC_KEY, INFO_VIOLATIONS_KEY, LINES_KEY, + MAJOR_VIOLATIONS_KEY, MINOR_VIOLATIONS_KEY, NCLOC_KEY, NCLOC_DATA_KEY, NCLOC_LANGUAGE_DISTRIBUTION_KEY, OPEN_ISSUES_KEY, RELIABILITY_RATING_KEY, + RELIABILITY_REMEDIATION_EFFORT_KEY, REOPENED_ISSUES_KEY, SECURITY_HOTSPOTS_KEY, SECURITY_HOTSPOTS_REVIEWED_KEY, SECURITY_HOTSPOTS_REVIEWED_STATUS_KEY, + SECURITY_HOTSPOTS_TO_REVIEW_STATUS_KEY, SECURITY_RATING_KEY, SECURITY_REMEDIATION_EFFORT_KEY, SECURITY_REVIEW_RATING_KEY, SQALE_DEBT_RATIO_KEY, TECHNICAL_DEBT_KEY, + SQALE_RATING_KEY, STATEMENTS_KEY, VIOLATIONS_KEY, VULNERABILITIES_KEY, WONT_FIX_ISSUES_KEY + ); private final DbClient dbClient; private final MetricRepository metricRepository; private final MeasureToMeasureDto measureToMeasureDto; private final TreeRootHolder treeRootHolder; private final MeasureRepository measureRepository; + private final Optional<FileStatuses> fileStatuses; public PersistLiveMeasuresStep(DbClient dbClient, MetricRepository metricRepository, MeasureToMeasureDto measureToMeasureDto, - TreeRootHolder treeRootHolder, MeasureRepository measureRepository) { + TreeRootHolder treeRootHolder, MeasureRepository measureRepository, Optional<FileStatuses> fileStatuses) { this.dbClient = dbClient; this.metricRepository = metricRepository; this.measureToMeasureDto = measureToMeasureDto; this.treeRootHolder = treeRootHolder; this.measureRepository = measureRepository; + this.fileStatuses = fileStatuses; } @Override @@ -99,11 +159,12 @@ public class PersistLiveMeasuresStep implements ComputationStep { @Override public void visitAny(Component component) { List<String> metricUuids = new ArrayList<>(); + List<String> keptMetricUuids = new ArrayList<>(); Map<String, Measure> measures = measureRepository.getRawMeasures(component); List<LiveMeasureDto> dtos = new ArrayList<>(); for (Map.Entry<String, Measure> measuresByMetricKey : measures.entrySet()) { String metricKey = measuresByMetricKey.getKey(); - if (NOT_TO_PERSIST_ON_FILE_METRIC_KEYS.contains(metricKey) && component.getType() == Component.Type.FILE) { + if (NOT_TO_PERSIST_ON_FILE_METRIC_KEYS.contains(metricKey) ) { continue; } Metric metric = metricRepository.getByKey(metricKey); @@ -112,27 +173,42 @@ public class PersistLiveMeasuresStep implements ComputationStep { if (!NonEmptyMeasure.INSTANCE.test(m) || !notBestValueOptimized.test(m)) { continue; } + metricUuids.add(metric.getUuid()); + if (shouldSkipMetricOnUnchangedFile(component, metricKey)) { + keptMetricUuids.add(metric.getUuid()); + continue; + } LiveMeasureDto lm = measureToMeasureDto.toLiveMeasureDto(m, metric, component); dtos.add(lm); - metricUuids.add(metric.getUuid()); } + List<String> excludedMetricUuids = supportUpsert ? metricUuids : keptMetricUuids; + deleteNonexistentMeasures(dbSession, component.getUuid(), excludedMetricUuids); + dtos.forEach(dto -> insertMeasureOptimally(dbSession, dto)); + + dbSession.commit(); + insertsOrUpdates += dtos.size(); + } + + private void deleteNonexistentMeasures(DbSession dbSession, String componentUuid, List<String> excludedMetricUuids) { + // The measures that no longer exist on the component must be deleted, for example + // when the coverage on a file goes to the "best value" 100%. + // The measures on deleted components are deleted by the step PurgeDatastoresStep + dbClient.liveMeasureDao().deleteByComponentUuidExcludingMetricUuids(dbSession, componentUuid, excludedMetricUuids); + } + + private void insertMeasureOptimally(DbSession dbSession, LiveMeasureDto dto) { if (supportUpsert) { - for (LiveMeasureDto dto : dtos) { - dbClient.liveMeasureDao().upsert(dbSession, dto); - } - // The measures that no longer exist on the component must be deleted, for example - // when the coverage on a file goes to the "best value" 100%. - // The measures on deleted components are deleted by the step PurgeDatastoresStep - dbClient.liveMeasureDao().deleteByComponentUuidExcludingMetricUuids(dbSession, component.getUuid(), metricUuids); + dbClient.liveMeasureDao().upsert(dbSession, dto); } else { - dbClient.liveMeasureDao().deleteByComponent(dbSession, component.getUuid()); - dtos.forEach(dto -> dbClient.liveMeasureDao().insert(dbSession, dto)); + dbClient.liveMeasureDao().insert(dbSession, dto); } + } - dbSession.commit(); - insertsOrUpdates += dtos.size(); + private boolean shouldSkipMetricOnUnchangedFile(Component component, String metricKey) { + return component.getType() == Component.Type.FILE && fileStatuses.isPresent() && + fileStatuses.get().isDataUnchanged(component) && NOT_TO_PERSIST_ON_UNCHANGED_FILES.contains(metricKey); } } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java index de5442922b0..a67ed8c74d9 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java @@ -50,6 +50,7 @@ public class ReportComputationSteps extends AbstractComputationSteps { LoadQualityProfilesStep.class, // load project related stuffs + LoadFileHashesAndStatusStep.class, LoadQualityGateStep.class, LoadPeriodsStep.class, FileMoveDetectionStep.class, diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilderTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilderTest.java index 80c02c98421..c5b115aa8a2 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilderTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilderTest.java @@ -553,6 +553,31 @@ public class ComponentTreeBuilderTest { } @Test + public void files_have_markedAsUnchanged_flag() { + ScannerReport.Component project = newBuilder() + .setType(PROJECT) + .setKey("c1") + .setRef(1) + .addChildRef(2) + .build(); + scannerComponentProvider.add(newBuilder() + .setRef(2) + .setType(FILE) + .setMarkedAsUnchanged(true) + .setProjectRelativePath("src/js/Foo.js") + .setLines(1)); + + Component root = call(project); + assertThat(root.getUuid()).isEqualTo("generated_c1_uuid"); + + Component directory = root.getChildren().iterator().next(); + assertThat(directory.getUuid()).isEqualTo("generated_c1:src/js_uuid"); + + Component file = directory.getChildren().iterator().next(); + assertThat(file.getFileAttributes().isMarkedAsUnchanged()).isTrue(); + } + + @Test public void issues_are_relocated_from_directories_and_modules_to_root() { ScannerReport.Component project = newBuilder() .setType(PROJECT) diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileAttributesTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileAttributesTest.java index cc7075e8916..1e70547c45e 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileAttributesTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileAttributesTest.java @@ -25,29 +25,29 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; public class FileAttributesTest { - - @Test public void create_production_file() { - FileAttributes underTest = new FileAttributes(true, "java", 10); + FileAttributes underTest = new FileAttributes(true, "java", 10, true); assertThat(underTest.isUnitTest()).isTrue(); assertThat(underTest.getLanguageKey()).isEqualTo("java"); assertThat(underTest.getLines()).isEqualTo(10); + assertThat(underTest.isMarkedAsUnchanged()).isTrue(); } @Test public void create_unit_test() { - FileAttributes underTest = new FileAttributes(true, "java", 10); + FileAttributes underTest = new FileAttributes(true, "java", 10, false); assertThat(underTest.isUnitTest()).isTrue(); assertThat(underTest.getLanguageKey()).isEqualTo("java"); assertThat(underTest.getLines()).isEqualTo(10); + assertThat(underTest.isMarkedAsUnchanged()).isFalse(); } @Test public void create_without_language() { - FileAttributes underTest = new FileAttributes(true, null, 10); + FileAttributes underTest = new FileAttributes(true, null, 10, false); assertThat(underTest.isUnitTest()).isTrue(); assertThat(underTest.getLanguageKey()).isNull(); @@ -56,21 +56,23 @@ public class FileAttributesTest { @Test public void fail_with_IAE_when_lines_is_0() { - assertThatThrownBy(() -> new FileAttributes(true, "java", 0)) + assertThatThrownBy(() -> new FileAttributes(true, "java", 0, false)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Number of lines must be greater than zero"); } @Test public void fail_with_IAE_when_lines_is_less_than_0() { - assertThatThrownBy(() -> new FileAttributes(true, "java", -10)) + assertThatThrownBy(() -> new FileAttributes(true, "java", -10, false)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Number of lines must be greater than zero"); } @Test public void test_toString() { - assertThat(new FileAttributes(true, "java", 10)).hasToString("FileAttributes{languageKey='java', unitTest=true, lines=10}"); - assertThat(new FileAttributes(false, null, 1)).hasToString("FileAttributes{languageKey='null', unitTest=false, lines=1}"); + assertThat(new FileAttributes(true, "java", 10, true)) + .hasToString("FileAttributes{languageKey='java', unitTest=true, lines=10, markedAsUnchanged=true}"); + assertThat(new FileAttributes(false, null, 1, false)) + .hasToString("FileAttributes{languageKey='null', unitTest=false, lines=1, markedAsUnchanged=false}"); } } diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java new file mode 100644 index 00000000000..3b181db1893 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java @@ -0,0 +1,184 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.component; + +import java.util.Optional; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.ce.task.projectanalysis.analysis.Analysis; +import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolderRule; +import org.sonar.ce.task.projectanalysis.source.SourceHashRepository; +import org.sonar.db.source.FileHashesDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class FileStatusesImplTest { + private static final String PROJECT_KEY = "PROJECT_KEY"; + private static final String PROJECT_UUID = "UUID-1234"; + + @Rule + public final TreeRootHolderRule treeRootHolder = new TreeRootHolderRule(); + private final PreviousSourceHashRepository previousSourceHashRepository = mock(PreviousSourceHashRepository.class); + private final SourceHashRepository sourceHashRepository = mock(SourceHashRepository.class); + @Rule + public final AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule(); + private final FileStatusesImpl fileStatuses = new FileStatusesImpl(analysisMetadataHolder, treeRootHolder, previousSourceHashRepository, sourceHashRepository); + + @Before + public void before() { + analysisMetadataHolder.setBaseAnalysis(new Analysis.Builder().setUuid(PROJECT_UUID).setCreatedAt(1000L).build()); + } + + @Test + public void file_is_unchanged_only_if_status_is_SAME_and_hashes_equal() { + Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME).build(); + Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build(); + Component file3 = ReportComponent.builder(Component.Type.FILE, 4, "FILE3_KEY").setStatus(Component.Status.CHANGED).build(); + + addDbFileHash(file1, "hash1"); + addDbFileHash(file2, "different"); + addDbFileHash(file3, "hash3"); + + addReportFileHash(file1, "hash1"); + addReportFileHash(file2, "hash2"); + addReportFileHash(file3, "hash3"); + + Component project = ReportComponent.builder(Component.Type.PROJECT, 1) + .setUuid(PROJECT_UUID) + .setKey(PROJECT_KEY) + .addChildren(file1, file2, file3) + .build(); + treeRootHolder.setRoot(project); + fileStatuses.initialize(); + assertThat(fileStatuses.isUnchanged(file1)).isTrue(); + assertThat(fileStatuses.isUnchanged(file2)).isFalse(); + assertThat(fileStatuses.isUnchanged(file2)).isFalse(); + } + + @Test + public void isDataUnchanged_returns_false_if_any_SAME_status_is_incorrect() { + Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME) + .setFileAttributes(new FileAttributes(false, null, 10, true)).build(); + Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build(); + + addDbFileHash(file1, "hash1"); + addDbFileHash(file2, "different"); + + addReportFileHash(file1, "hash1"); + addReportFileHash(file2, "hash2"); + + Component project = ReportComponent.builder(Component.Type.PROJECT, 1) + .setUuid(PROJECT_UUID) + .setKey(PROJECT_KEY) + .addChildren(file1, file2) + .build(); + treeRootHolder.setRoot(project); + fileStatuses.initialize(); + assertThat(fileStatuses.isDataUnchanged(file1)).isFalse(); + assertThat(fileStatuses.isDataUnchanged(file2)).isFalse(); + } + + @Test + public void isDataUnchanged_returns_false_no_previous_analysis() { + analysisMetadataHolder.setBaseAnalysis(null); + + Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME) + .setFileAttributes(new FileAttributes(false, null, 10, true)).build(); + Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build(); + + addReportFileHash(file1, "hash1"); + addReportFileHash(file2, "hash2"); + + Component project = ReportComponent.builder(Component.Type.PROJECT, 1) + .setUuid(PROJECT_UUID) + .setKey(PROJECT_KEY) + .addChildren(file1, file2) + .build(); + treeRootHolder.setRoot(project); + fileStatuses.initialize(); + + assertThat(fileStatuses.isDataUnchanged(file1)).isFalse(); + assertThat(fileStatuses.isDataUnchanged(file2)).isFalse(); + } + + @Test + public void isDataUnchanged_returns_false_if_not_set_by_analyzer() { + Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME) + .setFileAttributes(new FileAttributes(false, null, 10, false)).build(); + Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build(); + + addDbFileHash(file1, "hash1"); + addDbFileHash(file2, "hash2"); + + addReportFileHash(file1, "hash1"); + addReportFileHash(file2, "hash2"); + + Component project = ReportComponent.builder(Component.Type.PROJECT, 1) + .setUuid(PROJECT_UUID) + .setKey(PROJECT_KEY) + .addChildren(file1, file2) + .build(); + treeRootHolder.setRoot(project); + fileStatuses.initialize(); + assertThat(fileStatuses.isDataUnchanged(file1)).isFalse(); + assertThat(fileStatuses.isDataUnchanged(file2)).isFalse(); + } + + @Test + public void isDataUnchanged_returns_true_if_set_by_analyzer_and_all_SAME_status_are_correct() { + Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME) + .setFileAttributes(new FileAttributes(false, null, 10, true)).build(); + Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build(); + Component file3 = ReportComponent.builder(Component.Type.FILE, 4, "FILE3_KEY").setStatus(Component.Status.CHANGED).build(); + + addDbFileHash(file1, "hash1"); + addDbFileHash(file2, "hash2"); + addDbFileHash(file3, "hash3"); + + addReportFileHash(file1, "hash1"); + addReportFileHash(file2, "hash2"); + addReportFileHash(file3, "different"); + + Component project = ReportComponent.builder(Component.Type.PROJECT, 1) + .setUuid(PROJECT_UUID) + .setKey(PROJECT_KEY) + .addChildren(file1, file2, file3) + .build(); + treeRootHolder.setRoot(project); + fileStatuses.initialize(); + assertThat(fileStatuses.isDataUnchanged(file1)).isTrue(); + assertThat(fileStatuses.isDataUnchanged(file2)).isFalse(); + + verify(previousSourceHashRepository).getDbFile(file1); + } + + private void addDbFileHash(Component file, String hash) { + FileHashesDto fileHashesDto = new FileHashesDto().setSrcHash(hash); + when(previousSourceHashRepository.getDbFile(file)).thenReturn(Optional.of(fileHashesDto)); + } + + private void addReportFileHash(Component file, String hash) { + when(sourceHashRepository.getRawSourceHash(file)).thenReturn(hash); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImplTest.java new file mode 100644 index 00000000000..7ff9771fd48 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/PreviousSourceHashRepositoryImplTest.java @@ -0,0 +1,62 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.component; + +import java.util.Map; +import org.junit.Test; +import org.sonar.db.source.FileHashesDto; +import org.sonar.db.source.FileSourceDto; + +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +public class PreviousSourceHashRepositoryImplTest { + private final PreviousSourceHashRepositoryImpl previousFileHashesRepository = new PreviousSourceHashRepositoryImpl(); + + @Test + public void return_file_hashes() { + Component file1 = ReportComponent.builder(Component.Type.FILE, 1).build(); + Component file2 = ReportComponent.builder(Component.Type.FILE, 2).build(); + Component file3 = ReportComponent.builder(Component.Type.FILE, 3).build(); + + FileSourceDto fileSource1 = new FileSourceDto(); + FileSourceDto fileSource2 = new FileSourceDto(); + + previousFileHashesRepository.set(Map.of(file1.getUuid(), fileSource1, file2.getUuid(), fileSource2)); + assertThat(previousFileHashesRepository.getDbFile(file1)).contains(fileSource1); + assertThat(previousFileHashesRepository.getDbFile(file2)).contains(fileSource2); + assertThat(previousFileHashesRepository.getDbFile(file3)).isEmpty(); + } + + @Test + public void fail_if_not_set() { + assertThatThrownBy(() -> previousFileHashesRepository.getDbFile(mock(Component.class))).isInstanceOf(IllegalStateException.class); + } + + @Test + public void fail_if_set_twice() { + Map<String, FileHashesDto> empty = emptyMap(); + previousFileHashesRepository.set(empty); + assertThatThrownBy(() -> previousFileHashesRepository.set(empty)).isInstanceOf(IllegalStateException.class); + + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitorTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitorTest.java index 2f3ebe623f3..7e261b7ad72 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitorTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/IntegrateIssuesVisitorTest.java @@ -36,6 +36,7 @@ import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; import org.sonar.ce.task.projectanalysis.analysis.Branch; import org.sonar.ce.task.projectanalysis.batch.BatchReportReaderRule; import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.FileStatuses; import org.sonar.ce.task.projectanalysis.component.ReferenceBranchComponentUuids; import org.sonar.ce.task.projectanalysis.component.ReportComponent; import org.sonar.ce.task.projectanalysis.component.ReportModulesPath; @@ -50,7 +51,6 @@ import org.sonar.ce.task.projectanalysis.qualityprofile.AlwaysActiveRulesHolderI import org.sonar.ce.task.projectanalysis.source.NewLinesRepository; import org.sonar.ce.task.projectanalysis.source.SourceLinesHashRepository; import org.sonar.ce.task.projectanalysis.source.SourceLinesRepository; -import org.sonar.ce.task.projectanalysis.source.SourceLinesRepositoryRule; import org.sonar.core.issue.DefaultIssue; import org.sonar.core.issue.FieldDiffs; import org.sonar.core.issue.IssueChangeContext; @@ -113,8 +113,6 @@ public class IntegrateIssuesVisitorTest { public ActiveRulesHolderRule activeRulesHolderRule = new ActiveRulesHolderRule(); @Rule public RuleRepositoryRule ruleRepositoryRule = new RuleRepositoryRule(); - @Rule - public SourceLinesRepositoryRule fileSourceRepository = new SourceLinesRepositoryRule(); private final AnalysisMetadataHolder analysisMetadataHolder = mock(AnalysisMetadataHolder.class); private final IssueFilter issueFilter = mock(IssueFilter.class); @@ -128,8 +126,9 @@ public class IntegrateIssuesVisitorTest { private final ReferenceBranchComponentUuids referenceBranchComponentUuids = mock(ReferenceBranchComponentUuids.class); private final SourceLinesHashRepository sourceLinesHash = mock(SourceLinesHashRepository.class); private final NewLinesRepository newLinesRepository = mock(NewLinesRepository.class); - private TargetBranchComponentUuids targetBranchComponentUuids = mock(TargetBranchComponentUuids.class); + private final TargetBranchComponentUuids targetBranchComponentUuids = mock(TargetBranchComponentUuids.class); private final SourceLinesRepository sourceLinesRepository = mock(SourceLinesRepository.class); + private final FileStatuses fileStatuses = mock(FileStatuses.class); private ArgumentCaptor<DefaultIssue> defaultIssueCaptor; private final ComponentIssuesLoader issuesLoader = new ComponentIssuesLoader(dbTester.getDbClient(), ruleRepositoryRule, activeRulesHolderRule, new MapSettings().asConfig(), @@ -166,8 +165,8 @@ public class IntegrateIssuesVisitorTest { protoIssueCache = new ProtoIssueCache(temp.newFile(), System2.INSTANCE); when(issueFilter.accept(any(DefaultIssue.class), eq(FILE))).thenReturn(true); when(issueChangeContext.date()).thenReturn(new Date()); - underTest = new IntegrateIssuesVisitor(protoIssueCache, rawInputFactory, issueLifecycle, issueVisitors, trackingDelegator, issueStatusCopier, referenceBranchComponentUuids, - mock(PullRequestSourceBranchMerger.class)); + underTest = new IntegrateIssuesVisitor(protoIssueCache, rawInputFactory, baseInputFactory, issueLifecycle, issueVisitors, trackingDelegator, issueStatusCopier, + referenceBranchComponentUuids, mock(PullRequestSourceBranchMerger.class), fileStatuses); } @Test @@ -181,7 +180,6 @@ public class IntegrateIssuesVisitorTest { .setSeverity(Constants.Severity.BLOCKER) .build(); reportReader.putIssues(FILE_REF, singletonList(reportIssue)); - fileSourceRepository.addLine(FILE_REF, "line1"); underTest.visitAny(FILE); @@ -190,7 +188,6 @@ public class IntegrateIssuesVisitorTest { @Test public void process_existing_issue() { - RuleKey ruleKey = RuleTesting.XOO_X1; // Issue from db has severity major addBaseIssue(ruleKey); @@ -203,19 +200,16 @@ public class IntegrateIssuesVisitorTest { .setSeverity(Constants.Severity.BLOCKER) .build(); reportReader.putIssues(FILE_REF, singletonList(reportIssue)); - fileSourceRepository.addLine(FILE_REF, "line1"); underTest.visitAny(FILE); List<DefaultIssue> issues = newArrayList(protoIssueCache.traverse()); assertThat(issues).hasSize(1); assertThat(issues.get(0).severity()).isEqualTo(Severity.BLOCKER); - } @Test public void dont_cache_existing_issue_if_unmodified() { - RuleKey ruleKey = RuleTesting.XOO_X1; // Issue from db has severity major addBaseIssue(ruleKey); @@ -228,14 +222,12 @@ public class IntegrateIssuesVisitorTest { .setSeverity(Constants.Severity.BLOCKER) .build(); reportReader.putIssues(FILE_REF, singletonList(reportIssue)); - fileSourceRepository.addLine(FILE_REF, "line1"); underTest.visitAny(FILE); List<DefaultIssue> issues = newArrayList(protoIssueCache.traverse()); assertThat(issues).hasSize(1); assertThat(issues.get(0).severity()).isEqualTo(Severity.BLOCKER); - } @Test @@ -248,7 +240,6 @@ public class IntegrateIssuesVisitorTest { .setSeverity(Constants.Severity.BLOCKER) .build(); reportReader.putIssues(FILE_REF, singletonList(reportIssue)); - fileSourceRepository.addLine(FILE_REF, "line1"); underTest.visitAny(FILE); @@ -280,8 +271,34 @@ public class IntegrateIssuesVisitorTest { } @Test - public void copy_issues_when_creating_new_non_main_branch() { + public void reuse_issues_when_data_unchanged() { + RuleKey ruleKey = RuleTesting.XOO_X1; + // Issue from db has severity major + addBaseIssue(ruleKey); + + // Issue from report has severity blocker + ScannerReport.Issue reportIssue = ScannerReport.Issue.newBuilder() + .setMsg("new message") + .setRuleRepository(ruleKey.repository()) + .setRuleKey(ruleKey.rule()) + .setSeverity(Constants.Severity.BLOCKER) + .build(); + reportReader.putIssues(FILE_REF, singletonList(reportIssue)); + when(fileStatuses.isDataUnchanged(FILE)).thenReturn(true); + + underTest.visitAny(FILE); + // visitors get called, so measures created from issues should be calculated taking these issues into account + verify(issueVisitor).onIssue(eq(FILE), defaultIssueCaptor.capture()); + assertThat(defaultIssueCaptor.getValue().ruleKey().rule()).isEqualTo(ruleKey.rule()); + + // most issues won't go to the cache since they aren't changed and don't need to be persisted + // In this test they are being closed but the workflows aren't working (we mock them) so nothing is changed on the issue is not cached. + assertThat(newArrayList(protoIssueCache.traverse())).isEmpty(); + } + + @Test + public void copy_issues_when_creating_new_non_main_branch() { when(mergeBranchComponentsUuids.getComponentUuid(FILE_KEY)).thenReturn(FILE_UUID_ON_BRANCH); when(referenceBranchComponentUuids.getReferenceBranchName()).thenReturn("master"); @@ -304,7 +321,6 @@ public class IntegrateIssuesVisitorTest { .setSeverity(Constants.Severity.BLOCKER) .build(); reportReader.putIssues(FILE_REF, singletonList(reportIssue)); - fileSourceRepository.addLine(FILE_REF, "line1"); underTest.visitAny(FILE); diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java index 16c69b83a34..300d34bc83c 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java @@ -40,9 +40,9 @@ import org.sonar.ce.task.projectanalysis.component.Component; import org.sonar.ce.task.projectanalysis.component.Component.Status; import org.sonar.ce.task.projectanalysis.component.Component.Type; import org.sonar.ce.task.projectanalysis.component.FileAttributes; +import org.sonar.ce.task.projectanalysis.component.FileStatuses; import org.sonar.ce.task.projectanalysis.component.ReportComponent; import org.sonar.ce.task.projectanalysis.component.ViewsComponent; -import org.sonar.ce.task.projectanalysis.source.SourceHashRepository; import org.sonar.ce.task.projectanalysis.source.SourceLinesDiff; import org.sonar.db.protobuf.DbFileSources.Line; import org.sonar.scanner.protocol.output.ScannerReport; @@ -74,12 +74,11 @@ public class ScmInfoRepositoryImplTest { @Rule public AnalysisMetadataHolderRule analysisMetadata = new AnalysisMetadataHolderRule(); - private SourceHashRepository sourceHashRepository = mock(SourceHashRepository.class); - private SourceLinesDiff diff = mock(SourceLinesDiff.class); - private ScmInfoDbLoader dbLoader = mock(ScmInfoDbLoader.class); - private Date analysisDate = new Date(); - - private ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(reportReader, analysisMetadata, dbLoader, diff, sourceHashRepository); + private final FileStatuses fileStatuses = mock(FileStatuses.class); + private final SourceLinesDiff diff = mock(SourceLinesDiff.class); + private final ScmInfoDbLoader dbLoader = mock(ScmInfoDbLoader.class); + private final Date analysisDate = new Date(); + private final ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(reportReader, analysisMetadata, dbLoader, diff, fileStatuses); @Before public void setUp() { @@ -106,7 +105,7 @@ public class ScmInfoRepositoryImplTest { assertThat(logTester.logs(TRACE)).isEmpty(); verifyNoInteractions(dbLoader); - verifyNoInteractions(sourceHashRepository); + verifyNoInteractions(fileStatuses); verifyNoInteractions(diff); } @@ -125,32 +124,32 @@ public class ScmInfoRepositoryImplTest { assertThat(logTester.logs(TRACE)).containsOnly("Reading SCM info from report for file 'FILE_KEY'"); verifyNoInteractions(dbLoader); - verifyNoInteractions(sourceHashRepository); + verifyNoInteractions(fileStatuses); verifyNoInteractions(diff); } @Test public void read_from_DB_if_no_report_and_file_unchanged() { - createDbScmInfoWithOneLine("hash"); - when(sourceHashRepository.getRawSourceHash(FILE_SAME)).thenReturn("hash"); + createDbScmInfoWithOneLine(); + when(fileStatuses.isUnchanged(FILE_SAME)).thenReturn(true); // should clear revision and author ScmInfo scmInfo = underTest.getScmInfo(FILE_SAME).get(); assertThat(scmInfo.getAllChangesets()).hasSize(1); assertChangeset(scmInfo.getChangesetForLine(1), null, null, 10L); - verify(sourceHashRepository).getRawSourceHash(FILE_SAME); + verify(fileStatuses).isUnchanged(FILE_SAME); verify(dbLoader).getScmInfo(FILE_SAME); verifyNoMoreInteractions(dbLoader); - verifyNoMoreInteractions(sourceHashRepository); + verifyNoMoreInteractions(fileStatuses); verifyNoInteractions(diff); } @Test public void read_from_DB_if_no_report_and_file_unchanged_and_copyFromPrevious_is_true() { - createDbScmInfoWithOneLine("hash"); - when(sourceHashRepository.getRawSourceHash(FILE_SAME)).thenReturn("hash"); + createDbScmInfoWithOneLine(); + when(fileStatuses.isUnchanged(FILE_SAME)).thenReturn(true); addFileSourceInReport(1); addCopyFromPrevious(); @@ -158,11 +157,11 @@ public class ScmInfoRepositoryImplTest { assertThat(scmInfo.getAllChangesets()).hasSize(1); assertChangeset(scmInfo.getChangesetForLine(1), "rev1", "author1", 10L); - verify(sourceHashRepository).getRawSourceHash(FILE_SAME); + verify(fileStatuses).isUnchanged(FILE_SAME); verify(dbLoader).getScmInfo(FILE_SAME); verifyNoMoreInteractions(dbLoader); - verifyNoMoreInteractions(sourceHashRepository); + verifyNoMoreInteractions(fileStatuses); verifyNoInteractions(diff); } @@ -178,7 +177,7 @@ public class ScmInfoRepositoryImplTest { verify(dbLoader).getScmInfo(FILE); verifyNoMoreInteractions(dbLoader); - verifyNoInteractions(sourceHashRepository); + verifyNoInteractions(fileStatuses); verifyNoInteractions(diff); } @@ -195,13 +194,13 @@ public class ScmInfoRepositoryImplTest { verify(dbLoader).getScmInfo(FILE); verifyNoMoreInteractions(dbLoader); - verifyNoInteractions(sourceHashRepository); + verifyNoInteractions(fileStatuses); verifyNoInteractions(diff); } @Test public void generate_scm_info_for_new_and_changed_lines_when_report_is_empty() { - createDbScmInfoWithOneLine("hash"); + createDbScmInfoWithOneLine(); when(diff.computeMatchingLines(FILE)).thenReturn(new int[] {1, 0, 0}); addFileSourceInReport(3); ScmInfo scmInfo = underTest.getScmInfo(FILE).get(); @@ -214,7 +213,6 @@ public class ScmInfoRepositoryImplTest { verify(dbLoader).getScmInfo(FILE); verify(diff).computeMatchingLines(FILE); verifyNoMoreInteractions(dbLoader); - verifyNoInteractions(sourceHashRepository); verifyNoMoreInteractions(diff); } @@ -229,7 +227,7 @@ public class ScmInfoRepositoryImplTest { @UseDataProvider("allTypeComponentButFile") public void do_not_query_db_nor_report_if_component_type_is_not_FILE(Component component) { BatchReportReader batchReportReader = mock(BatchReportReader.class); - ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(batchReportReader, analysisMetadata, dbLoader, diff, sourceHashRepository); + ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(batchReportReader, analysisMetadata, dbLoader, diff, fileStatuses); assertThat(underTest.getScmInfo(component)).isEmpty(); @@ -278,13 +276,13 @@ public class ScmInfoRepositoryImplTest { reportReader.putChangesets(Changesets.newBuilder().setComponentRef(FILE_REF).setCopyFromPrevious(true).build()); } - private DbScmInfo createDbScmInfoWithOneLine(String hash) { + private DbScmInfo createDbScmInfoWithOneLine() { Line line1 = Line.newBuilder().setLine(1) .setScmRevision("rev1") .setScmAuthor("author1") .setScmDate(10L) .build(); - DbScmInfo scmInfo = DbScmInfo.create(Collections.singletonList(line1), 1, hash).get(); + DbScmInfo scmInfo = DbScmInfo.create(Collections.singletonList(line1), 1, "hash1").get(); when(dbLoader.getScmInfo(FILE)).thenReturn(Optional.of(scmInfo)); return scmInfo; } diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStepTest.java index c7234e89aa2..105c5d20226 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/PersistFileSourcesStepTest.java @@ -22,15 +22,16 @@ package org.sonar.ce.task.projectanalysis.source; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.function.Consumer; import org.apache.commons.codec.digest.DigestUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.mockito.Mockito; import org.sonar.api.utils.System2; import org.sonar.ce.task.projectanalysis.component.Component; import org.sonar.ce.task.projectanalysis.component.FileAttributes; +import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepository; import org.sonar.ce.task.projectanalysis.component.ReportComponent; import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; import org.sonar.ce.task.projectanalysis.scm.Changeset; @@ -43,10 +44,12 @@ import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.protobuf.DbFileSources; +import org.sonar.db.source.FileHashesDto; import org.sonar.db.source.FileSourceDto; import org.sonar.db.source.LineHashVersion; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -60,28 +63,30 @@ public class PersistFileSourcesStepTest extends BaseStepTest { private static final long NOW = 123456789L; private static final long PAST = 15000L; - private System2 system2 = mock(System2.class); + private final System2 system2 = mock(System2.class); @Rule public DbTester dbTester = DbTester.create(system2); @Rule public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule(); - private SourceLinesHashRepository sourceLinesHashRepository = mock(SourceLinesHashRepository.class); - private SourceLinesHashRepositoryImpl.LineHashesComputer lineHashesComputer = mock(SourceLinesHashRepositoryImpl.LineHashesComputer.class); - private FileSourceDataComputer fileSourceDataComputer = mock(FileSourceDataComputer.class); - private FileSourceDataWarnings fileSourceDataWarnings = mock(FileSourceDataWarnings.class); + private final SourceLinesHashRepository sourceLinesHashRepository = mock(SourceLinesHashRepository.class); + private final SourceLinesHashRepositoryImpl.LineHashesComputer lineHashesComputer = mock(SourceLinesHashRepositoryImpl.LineHashesComputer.class); + private final FileSourceDataComputer fileSourceDataComputer = mock(FileSourceDataComputer.class); + private final FileSourceDataWarnings fileSourceDataWarnings = mock(FileSourceDataWarnings.class); + private final PreviousSourceHashRepository previousSourceHashRepository = mock(PreviousSourceHashRepository.class); - private DbClient dbClient = dbTester.getDbClient(); - private DbSession session = dbTester.getSession(); + private final DbClient dbClient = dbTester.getDbClient(); + private final DbSession session = dbTester.getSession(); private PersistFileSourcesStep underTest; @Before public void setup() { when(system2.now()).thenReturn(NOW); - when(sourceLinesHashRepository.getLineHashesComputerToPersist(Mockito.any(Component.class))).thenReturn(lineHashesComputer); - underTest = new PersistFileSourcesStep(dbClient, system2, treeRootHolder, sourceLinesHashRepository, fileSourceDataComputer, fileSourceDataWarnings, new SequenceUuidFactory()); + when(sourceLinesHashRepository.getLineHashesComputerToPersist(any(Component.class))).thenReturn(lineHashesComputer); + underTest = new PersistFileSourcesStep(dbClient, system2, treeRootHolder, sourceLinesHashRepository, fileSourceDataComputer, fileSourceDataWarnings, + new SequenceUuidFactory(), previousSourceHashRepository); initBasicReport(1); } @@ -305,6 +310,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest { @Test public void not_update_sources_when_nothing_has_changed() { + setPastAnalysisHashes(); dbClient.fileSourceDao().insert(dbTester.getSession(), createDto()); dbTester.getSession().commit(); @@ -326,7 +332,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest { public void update_sources_when_source_updated() { // Existing sources long past = 150000L; - dbClient.fileSourceDao().insert(dbTester.getSession(), new FileSourceDto() + FileSourceDto dbFileSources = new FileSourceDto() .setUuid(Uuids.createFast()) .setProjectUuid(PROJECT_UUID) .setFileUuid(FILE1_UUID) @@ -341,8 +347,10 @@ public class PersistFileSourcesStepTest extends BaseStepTest { .build()) .setCreatedAt(past) .setUpdatedAt(past) - .setRevision("rev-0")); + .setRevision("rev-0"); + dbClient.fileSourceDao().insert(dbTester.getSession(), dbFileSources); dbTester.getSession().commit(); + setPastAnalysisHashes(dbFileSources); DbFileSources.Data newSourceData = DbFileSources.Data.newBuilder() .addLines(DbFileSources.Line.newBuilder() @@ -369,8 +377,10 @@ public class PersistFileSourcesStepTest extends BaseStepTest { @Test public void update_sources_when_src_hash_is_missing() { - dbClient.fileSourceDao().insert(dbTester.getSession(), createDto(dto -> dto.setSrcHash(null))); + FileSourceDto dbFileSources = createDto(dto -> dto.setSrcHash(null)); + dbClient.fileSourceDao().insert(dbTester.getSession(), dbFileSources); dbTester.getSession().commit(); + setPastAnalysisHashes(dbFileSources); DbFileSources.Data sourceData = DbFileSources.Data.newBuilder().build(); setComputedData(sourceData, Collections.singletonList("lineHash"), "newSourceHash", null); @@ -394,8 +404,10 @@ public class PersistFileSourcesStepTest extends BaseStepTest { .build()) .build(); - dbClient.fileSourceDao().insert(dbTester.getSession(), createDto(dto -> dto.setRevision(null))); + FileSourceDto dbFileSources = createDto(dto -> dto.setRevision(null)); + dbClient.fileSourceDao().insert(dbTester.getSession(), dbFileSources); dbTester.getSession().commit(); + setPastAnalysisHashes(dbFileSources); Changeset changeset = Changeset.newChangesetBuilder().setDate(1L).setRevision("revision").build(); setComputedData(sourceData, Collections.singletonList("137f72c3708c6bd0de00a0e5a69c699b"), "29f25900140c94db38035128cb6de6a2", changeset); @@ -436,6 +448,21 @@ public class PersistFileSourcesStepTest extends BaseStepTest { return dto; } + private void setPastAnalysisHashes() { + DbFileSources.Data sourceData = DbFileSources.Data.newBuilder().build(); + byte[] data = FileSourceDto.encodeSourceData(sourceData); + String dataHash = DigestUtils.md5Hex(data); + FileHashesDto fileHashesDto = new FileHashesDto() + .setSrcHash("sourceHash") + .setDataHash(dataHash) + .setRevision("rev-1"); + setPastAnalysisHashes(fileHashesDto); + } + + private void setPastAnalysisHashes(FileHashesDto fileHashesDto) { + when(previousSourceHashRepository.getDbFile(any(Component.class))).thenReturn(Optional.of(fileHashesDto)); + } + private void setComputedData(DbFileSources.Data data, List<String> lineHashes, String sourceHash, Changeset latestChangeWithRevision) { FileSourceDataComputer.Data computedData = new FileSourceDataComputer.Data(data, lineHashes, sourceHash, latestChangeWithRevision); when(fileSourceDataComputer.compute(fileComponent().build(), fileSourceDataWarnings)).thenReturn(computedData); diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStepTest.java new file mode 100644 index 00000000000..68bb25db7b1 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadFileHashesAndStatusStepTest.java @@ -0,0 +1,126 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.util.ArrayList; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.FileStatusesImpl; +import org.sonar.ce.task.projectanalysis.component.PreviousSourceHashRepositoryImpl; +import org.sonar.ce.task.projectanalysis.component.ReportComponent; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ComponentTesting; +import org.sonar.db.source.FileHashesDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class LoadFileHashesAndStatusStepTest { + @Rule + public DbTester db = DbTester.create(); + @Rule + public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule(); + public PreviousSourceHashRepositoryImpl previousFileHashesRepository = new PreviousSourceHashRepositoryImpl(); + public FileStatusesImpl fileStatuses = mock(FileStatusesImpl.class); + public AnalysisMetadataHolder analysisMetadataHolder = mock(AnalysisMetadataHolder.class); + + private final LoadFileHashesAndStatusStep underTest = new LoadFileHashesAndStatusStep(db.getDbClient(), previousFileHashesRepository, fileStatuses, + db.getDbClient().fileSourceDao(), treeRootHolder); + + @Before + public void before() { + when(analysisMetadataHolder.isFirstAnalysis()).thenReturn(false); + } + + @Test + public void initialized_file_statuses() { + Component project = ReportComponent.builder(Component.Type.PROJECT, 1, "project").build(); + treeRootHolder.setRoot(project); + underTest.execute(new TestComputationStepContext()); + verify(fileStatuses).initialize(); + } + + @Test + public void loads_file_hashes_for_project_branch() { + ComponentDto project1 = db.components().insertPublicProject(); + ComponentDto project2 = db.components().insertPublicProject(); + + ComponentDto dbFile1 = db.components().insertComponent(ComponentTesting.newFileDto(project1)); + ComponentDto dbFile2 = db.components().insertComponent(ComponentTesting.newFileDto(project1)); + + insertFileSources(dbFile1, dbFile2); + + Component reportFile1 = ReportComponent.builder(Component.Type.FILE, 2, dbFile1.getKey()).setUuid(dbFile1.uuid()).build(); + Component reportFile2 = ReportComponent.builder(Component.Type.FILE, 3, dbFile2.getKey()).setUuid(dbFile2.uuid()).build(); + Component reportFile3 = ReportComponent.builder(Component.Type.FILE, 4, dbFile2.getKey()).build(); + + treeRootHolder.setRoot(ReportComponent.builder(Component.Type.PROJECT, 1, project1.getKey()).setUuid(project1.uuid()) + .addChildren(reportFile1, reportFile2, reportFile3).build()); + underTest.execute(new TestComputationStepContext()); + + assertThat(previousFileHashesRepository.getMap()).hasSize(2); + assertThat(previousFileHashesRepository.getDbFile(reportFile1).get()) + .extracting(FileHashesDto::getSrcHash, FileHashesDto::getRevision, FileHashesDto::getDataHash) + .containsOnly("srcHash" + dbFile1.getKey(), "revision" + dbFile1.getKey(), "dataHash" + dbFile1.getKey()); + assertThat(previousFileHashesRepository.getDbFile(reportFile2).get()) + .extracting(FileHashesDto::getSrcHash, FileHashesDto::getRevision, FileHashesDto::getDataHash) + .containsOnly("srcHash" + dbFile2.getKey(), "revision" + dbFile2.getKey(), "dataHash" + dbFile2.getKey()); + assertThat(previousFileHashesRepository.getDbFile(reportFile3)).isEmpty(); + } + + @Test + public void loads_high_number_of_files() { + ComponentDto project1 = db.components().insertPublicProject(); + List<Component> files = new ArrayList<>(2000); + + for (int i = 0; i < 2000; i++) { + ComponentDto dbFile = db.components().insertComponent(ComponentTesting.newFileDto(project1)); + insertFileSources(dbFile); + files.add(ReportComponent.builder(Component.Type.FILE, 2, dbFile.getKey()).setUuid(dbFile.uuid()).build()); + } + + treeRootHolder.setRoot(ReportComponent.builder(Component.Type.PROJECT, 1, project1.getKey()).setUuid(project1.uuid()) + .addChildren(files.toArray(Component[]::new)).build()); + underTest.execute(new TestComputationStepContext()); + + assertThat(previousFileHashesRepository.getMap()).hasSize(2000); + for (Component file : files) { + assertThat(previousFileHashesRepository.getDbFile(file)).isPresent(); + } + } + + private void insertFileSources(ComponentDto... files) { + for (ComponentDto file : files) { + db.fileSources().insertFileSource(file, f -> f + .setSrcHash("srcHash" + file.getKey()) + .setRevision("revision" + file.getKey()) + .setDataHash("dataHash" + file.getKey())); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java index 47ca82a48ab..be8adf493fc 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistLiveMeasuresStepTest.java @@ -27,6 +27,7 @@ import org.sonar.api.measures.Metric; import org.sonar.api.utils.System2; import org.sonar.ce.task.projectanalysis.analysis.MutableAnalysisMetadataHolderRule; import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.FileStatuses; import org.sonar.ce.task.projectanalysis.component.ReportComponent; import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; import org.sonar.ce.task.projectanalysis.component.ViewsComponent; @@ -44,6 +45,10 @@ import org.sonar.server.project.Project; import static java.util.Collections.emptyList; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.api.measures.CoreMetrics.BUGS; import static org.sonar.ce.task.projectanalysis.component.Component.Type.DIRECTORY; import static org.sonar.ce.task.projectanalysis.component.Component.Type.FILE; import static org.sonar.ce.task.projectanalysis.component.Component.Type.PROJECT; @@ -78,7 +83,9 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { @Rule public MutableAnalysisMetadataHolderRule analysisMetadataHolder = new MutableAnalysisMetadataHolderRule(); - private DbClient dbClient = db.getDbClient(); + private final FileStatuses fileStatuses = mock(FileStatuses.class); + private final DbClient dbClient = db.getDbClient(); + private final TestComputationStepContext context = new TestComputationStepContext(); @Before public void setUp() { @@ -86,9 +93,11 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { MetricDto intMetricDto = db.measures().insertMetric(m -> m.setKey(INT_METRIC.getKey()).setValueType(Metric.ValueType.INT.name())); MetricDto bestValueMMetricDto = db.measures() .insertMetric(m -> m.setKey(METRIC_WITH_BEST_VALUE.getKey()).setValueType(Metric.ValueType.INT.name()).setOptimizedBestValue(true).setBestValue(0.0)); + MetricDto bugs = db.measures().insertMetric(m -> m.setKey(BUGS.getKey())); metricRepository.add(stringMetricDto.getUuid(), STRING_METRIC); metricRepository.add(intMetricDto.getUuid(), INT_METRIC); metricRepository.add(bestValueMMetricDto.getUuid(), METRIC_WITH_BEST_VALUE); + metricRepository.add(bugs.getUuid(), BUGS); } @Test @@ -100,7 +109,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("dir-value")); measureRepository.addRawMeasure(REF_4, STRING_METRIC.getKey(), newMeasureBuilder().create("file-value")); - TestComputationStepContext context = new TestComputationStepContext(); step().execute(context); // all measures are persisted, from project to file @@ -117,7 +125,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { measureRepository.addRawMeasure(REF_1, STRING_METRIC.getKey(), newMeasureBuilder().createNoValue()); measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().createNoValue()); - TestComputationStepContext context = new TestComputationStepContext(); step().execute(context); assertThatMeasureIsNotPersisted("project-uuid", STRING_METRIC); @@ -130,7 +137,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { prepareProject(); measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().setVariation(42.0).createNoValue()); - TestComputationStepContext context = new TestComputationStepContext(); step().execute(context); LiveMeasureDto persistedMeasure = selectMeasure("project-uuid", INT_METRIC).get(); @@ -152,7 +158,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { measureRepository.addRawMeasure(REF_4, INT_METRIC.getKey(), newMeasureBuilder().create(42)); - TestComputationStepContext context = new TestComputationStepContext(); step().execute(context); assertThatMeasureHasValue(measureOnFileInProject, 42); @@ -173,7 +178,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { // file measure with metric best value -> do not persist measureRepository.addRawMeasure(REF_4, METRIC_WITH_BEST_VALUE.getKey(), newMeasureBuilder().create(0)); - TestComputationStepContext context = new TestComputationStepContext(); step().execute(context); assertThatMeasureDoesNotExist(oldMeasure); @@ -182,6 +186,30 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { } @Test + public void keep_measures_for_unchanged_files() { + prepareProject(); + LiveMeasureDto oldMeasure = insertMeasure("file-uuid", "project-uuid", BUGS); + db.commit(); + when(fileStatuses.isDataUnchanged(any(Component.class))).thenReturn(true); + // this new value won't be persisted + measureRepository.addRawMeasure(REF_4, BUGS.getKey(), newMeasureBuilder().create(oldMeasure.getValue() + 1, 0)); + step().execute(context); + assertThat(selectMeasure("file-uuid", BUGS).get().getValue()).isEqualTo(oldMeasure.getValue()); + } + + @Test + public void dont_keep_measures_for_unchanged_files() { + prepareProject(); + LiveMeasureDto oldMeasure = insertMeasure("file-uuid", "project-uuid", BUGS); + db.commit(); + when(fileStatuses.isDataUnchanged(any(Component.class))).thenReturn(false); + // this new value will be persisted + measureRepository.addRawMeasure(REF_4, BUGS.getKey(), newMeasureBuilder().create(oldMeasure.getValue() + 1, 0)); + step().execute(context); + assertThat(selectMeasure("file-uuid", BUGS).get().getValue()).isEqualTo(oldMeasure.getValue() + 1); + } + + @Test public void persist_live_measures_of_portfolio_analysis() { preparePortfolio(); @@ -190,7 +218,6 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { measureRepository.addRawMeasure(REF_2, STRING_METRIC.getKey(), newMeasureBuilder().create("subview-value")); measureRepository.addRawMeasure(REF_3, STRING_METRIC.getKey(), newMeasureBuilder().create("project-value")); - TestComputationStepContext context = new TestComputationStepContext(); step().execute(context); assertThat(db.countRowsOfTable("live_measures")).isEqualTo(3); @@ -288,7 +315,8 @@ public class PersistLiveMeasuresStepTest extends BaseStepTest { @Override protected ComputationStep step() { - return new PersistLiveMeasuresStep(dbClient, metricRepository, new MeasureToMeasureDto(analysisMetadataHolder, treeRootHolder), treeRootHolder, measureRepository); + return new PersistLiveMeasuresStep(dbClient, metricRepository, new MeasureToMeasureDto(analysisMetadataHolder, treeRootHolder), treeRootHolder, measureRepository, + Optional.of(fileStatuses)); } private static void verifyStatistics(TestComputationStepContext context, int expectedInsertsOrUpdates) { diff --git a/server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/component/ReportComponent.java b/server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/component/ReportComponent.java index 575bcfeb006..5bcbcd9d11c 100644 --- a/server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/component/ReportComponent.java +++ b/server/sonar-ce-task-projectanalysis/src/testFixtures/java/org/sonar/ce/task/projectanalysis/component/ReportComponent.java @@ -36,7 +36,7 @@ import static java.util.Objects.requireNonNull; */ public class ReportComponent implements Component { - private static final FileAttributes DEFAULT_FILE_ATTRIBUTES = new FileAttributes(false, null, 1); + private static final FileAttributes DEFAULT_FILE_ATTRIBUTES = new FileAttributes(false, null, 1, false); public static final Component DUMB_PROJECT = builder(Type.PROJECT, 1) .setKey("PROJECT_KEY") diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileHashesDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileHashesDto.java new file mode 100644 index 00000000000..456f1979b5b --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileHashesDto.java @@ -0,0 +1,92 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.source; + +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; + +public class FileHashesDto { + protected String uuid; + protected String fileUuid; + protected String srcHash; + protected String dataHash; + protected String revision; + protected long updatedAt; + @Nullable + protected Integer lineHashesVersion; + + public int getLineHashesVersion() { + return lineHashesVersion != null ? lineHashesVersion : LineHashVersion.WITHOUT_SIGNIFICANT_CODE.getDbValue(); + } + + public String getUuid() { + return uuid; + } + + public String getFileUuid() { + return fileUuid; + } + + public FileHashesDto setFileUuid(String fileUuid) { + this.fileUuid = fileUuid; + return this; + } + @CheckForNull + public String getDataHash() { + return dataHash; + } + + + + /** + * MD5 of column BINARY_DATA. Used to know to detect data changes and need for update. + */ + public FileHashesDto setDataHash(String s) { + this.dataHash = s; + return this; + } + + @CheckForNull + public String getSrcHash() { + return srcHash; + } + + /** + * Hash of file content. Value is computed by batch. + */ + public FileHashesDto setSrcHash(@Nullable String srcHash) { + this.srcHash = srcHash; + return this; + } + + public String getRevision() { + return revision; + } + + public FileHashesDto setRevision(@Nullable String revision) { + this.revision = revision; + return this; + } + + public long getUpdatedAt() { + return updatedAt; + } + +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java index a2c4324e266..42c276ba030 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java @@ -50,6 +50,13 @@ public class FileSourceDao implements Dao { return version == null ? null : LineHashVersion.valueOf(version); } + /** + * The returning object doesn't contain all fields filled. For example, binary data is not loaded. + */ + public void scrollFileHashesByProjectUuid(DbSession dbSession, String projectUuid, ResultHandler<FileHashesDto> rowHandler) { + mapper(dbSession).scrollHashesForProject(projectUuid, rowHandler); + } + @CheckForNull public List<String> selectLineHashes(DbSession dbSession, String fileUuid) { Connection connection = dbSession.getConnection(); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDto.java index 7b92a27d47e..e4525cec791 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDto.java @@ -28,7 +28,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Collections; import java.util.List; -import javax.annotation.CheckForNull; import javax.annotation.Nullable; import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; @@ -38,7 +37,7 @@ import org.sonar.db.protobuf.DbFileSources; import static com.google.common.base.Splitter.on; import static java.lang.String.format; -public class FileSourceDto { +public class FileSourceDto extends FileHashesDto { private static final String SIZE_LIMIT_EXCEEDED_EXCEPTION_MESSAGE = "Protocol message was too large. May be malicious. " + "Use CodedInputStream.setSizeLimit() to increase the size limit."; @@ -46,11 +45,8 @@ public class FileSourceDto { public static final Splitter LINES_HASHES_SPLITTER = on('\n'); public static final int LINE_COUNT_NOT_POPULATED = -1; - private String uuid; private String projectUuid; - private String fileUuid; private long createdAt; - private long updatedAt; private String lineHashes; /** * When {@code line_count} column has been added, it's been populated with value {@link #LINE_COUNT_NOT_POPULATED -1}, @@ -64,26 +60,13 @@ public class FileSourceDto { * {@code line_hashes}. */ private int lineCount = LINE_COUNT_NOT_POPULATED; - private String srcHash; private byte[] binaryData = new byte[0]; - private String dataHash; - private String revision; - @Nullable - private Integer lineHashesVersion; - - public int getLineHashesVersion() { - return lineHashesVersion != null ? lineHashesVersion : LineHashVersion.WITHOUT_SIGNIFICANT_CODE.getDbValue(); - } public FileSourceDto setLineHashesVersion(int lineHashesVersion) { this.lineHashesVersion = lineHashesVersion; return this; } - public String getUuid() { - return uuid; - } - public FileSourceDto setUuid(String uuid) { this.uuid = uuid; return this; @@ -98,20 +81,11 @@ public class FileSourceDto { return this; } - public String getFileUuid() { - return fileUuid; - } - public FileSourceDto setFileUuid(String fileUuid) { this.fileUuid = fileUuid; return this; } - @CheckForNull - public String getDataHash() { - return dataHash; - } - /** * MD5 of column BINARY_DATA. Used to know to detect data changes and need for update. */ @@ -234,11 +208,6 @@ public class FileSourceDto { return this; } - @CheckForNull - public String getSrcHash() { - return srcHash; - } - /** * Hash of file content. Value is computed by batch. */ @@ -256,19 +225,11 @@ public class FileSourceDto { return this; } - public long getUpdatedAt() { - return updatedAt; - } - public FileSourceDto setUpdatedAt(long updatedAt) { this.updatedAt = updatedAt; return this; } - public String getRevision() { - return revision; - } - public FileSourceDto setRevision(@Nullable String revision) { this.revision = revision; return this; diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java index 68580975c11..a77fb535216 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java @@ -20,14 +20,13 @@ package org.sonar.db.source; import java.util.Collection; -import java.util.List; import javax.annotation.CheckForNull; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.session.ResultHandler; public interface FileSourceMapper { - List<FileSourceDto> selectHashesForProject(@Param("projectUuid") String projectUuid); + void scrollHashesForProject(@Param("projectUuid") String projectUuid, ResultHandler<FileHashesDto> rowHandler); @CheckForNull FileSourceDto selectByFileUuid(@Param("fileUuid") String fileUuid); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml index 608faddf398..53bba0f8365 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml @@ -24,14 +24,15 @@ file_uuid = #{fileUuid,jdbcType=VARCHAR} </select> - <select id="selectHashesForProject" parameterType="map" resultType="org.sonar.db.source.FileSourceDto"> + <select id="scrollHashesForProject" parameterType="map" resultType="org.sonar.db.source.FileHashesDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY"> select uuid, file_uuid as fileUuid, data_hash as dataHash, src_hash as srcHash, revision, - updated_at as updatedAt + updated_at as updatedAt, + line_hashes_version as lineHashesVersion from file_sources where diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java index aabd15fe3f4..885ed6d3bde 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java @@ -22,7 +22,9 @@ package org.sonar.db.source; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Random; import java.util.stream.Collectors; @@ -232,6 +234,43 @@ public class FileSourceDaoTest { } @Test + public void scrollFileHashes_handles_scrolling_more_than_1000_files() { + ComponentDto project = dbTester.components().insertPrivateProject(); + List<ComponentDto> files = IntStream.range(0, 1001 + new Random().nextInt(5)) + .mapToObj(i -> { + ComponentDto file = dbTester.components().insertComponent(newFileDto(project)); + dbTester.fileSources().insertFileSource(file); + return file; + }) + .collect(Collectors.toList()); + + Map<String, FileHashesDto> fileSourcesByUuid = new HashMap<>(); + underTest.scrollFileHashesByProjectUuid(dbSession, project.projectUuid(), result -> fileSourcesByUuid.put(result.getResultObject().getFileUuid(), result.getResultObject())); + + assertThat(fileSourcesByUuid).hasSize(files.size()); + files.forEach(t -> assertThat(fileSourcesByUuid).containsKey(t.uuid())); + } + + @Test + public void scrollFileHashes_returns_all_hashes() { + ComponentDto project = dbTester.components().insertPrivateProject(); + ComponentDto file = dbTester.components().insertComponent(newFileDto(project)); + FileSourceDto inserted = dbTester.fileSources().insertFileSource(file); + + List<FileHashesDto> fileSources = new ArrayList<>(1); + underTest.scrollFileHashesByProjectUuid(dbSession, project.projectUuid(), result -> fileSources.add(result.getResultObject())); + + assertThat(fileSources).hasSize(1); + FileHashesDto fileSource = fileSources.iterator().next(); + + assertThat(fileSource.getDataHash()).isEqualTo(inserted.getDataHash()); + assertThat(fileSource.getFileUuid()).isEqualTo(inserted.getFileUuid()); + assertThat(fileSource.getRevision()).isEqualTo(inserted.getRevision()); + assertThat(fileSource.getSrcHash()).isEqualTo(inserted.getSrcHash()); + assertThat(fileSource.getLineHashesVersion()).isEqualTo(inserted.getLineHashesVersion()); + } + + @Test public void scrollLineHashes_handles_scrolling_more_than_1000_files() { ComponentDto project = new Random().nextBoolean() ? dbTester.components().insertPrivateProject() : dbTester.components().insertPublicProject(); List<ComponentDto> files = IntStream.range(0, 1001 + new Random().nextInt(5)) diff --git a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputComponent.java b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputComponent.java index cb37d133823..431575f1b54 100644 --- a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputComponent.java +++ b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputComponent.java @@ -52,6 +52,7 @@ public abstract class DefaultInputComponent implements InputComponent { return id; } + @Override public int hashCode() { return key().hashCode(); @@ -68,5 +69,6 @@ public abstract class DefaultInputComponent implements InputComponent { public boolean hasMeasureFor(Metric metric) { return storedMetricKeys.contains(metric.key()); + } } diff --git a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java index 888763dfdb7..382461fbc5a 100644 --- a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java +++ b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java @@ -70,6 +70,8 @@ public class DefaultInputFile extends DefaultInputComponent implements InputFile private Metadata metadata; private Collection<int[]> ignoreIssuesOnlineRanges; private BitSet executableLines; + private boolean markedAsUnchanged; + public DefaultInputFile(DefaultIndexedFile indexedFile, Consumer<DefaultInputFile> metadataGenerator) { this(indexedFile, metadataGenerator, null); @@ -81,6 +83,7 @@ public class DefaultInputFile extends DefaultInputComponent implements InputFile this.indexedFile = indexedFile; this.metadataGenerator = metadataGenerator; this.metadata = null; + this.markedAsUnchanged = false; this.published = false; this.excludedForCoverage = false; this.contents = contents; @@ -99,6 +102,15 @@ public class DefaultInputFile extends DefaultInputComponent implements InputFile ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE); } + public boolean isMarkedAsUnchanged() { + return markedAsUnchanged; + } + + public DefaultInputComponent setMarkedAsUnchanged(boolean markedAsUnchanged) { + this.markedAsUnchanged = markedAsUnchanged; + return this; + } + @Override public String contents() throws IOException { if (contents != null) { diff --git a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java index 98b48f12382..5c03698dc68 100644 --- a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java +++ b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java @@ -414,6 +414,11 @@ public class SensorContextTester implements SensorContext { } @Override + public void markAsUnchanged(InputFile inputFile) { + ((DefaultInputFile) inputFile).setMarkedAsUnchanged(true); + } + + @Override public WriteCache nextCache() { return writeCache; } @@ -427,7 +432,6 @@ public class SensorContextTester implements SensorContext { return readCache; } - public void setPreviousCache(ReadCache cache) { this.readCache = cache; } @@ -437,7 +441,6 @@ public class SensorContextTester implements SensorContext { return cacheEnabled; } - public void setCacheEnabled(boolean enabled) { this.cacheEnabled = enabled; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java index 9fe8fc5a7f5..4ca7c8be780 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java @@ -66,6 +66,7 @@ public class ComponentsPublisher implements ReportPublisherStep { fileBuilder.setIsTest(file.type() == InputFile.Type.TEST); fileBuilder.setLines(file.lines()); fileBuilder.setStatus(convert(file.status())); + fileBuilder.setMarkedAsUnchanged(file.isMarkedAsUnchanged()); String lang = getLanguageKey(file); if (lang != null) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java index 7aaf4b5eebe..f2c25f372e4 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java @@ -186,6 +186,10 @@ public class DefaultSensorStorage implements SensorStorage { reportPublisher.getWriter().appendComponentMeasure(((DefaultInputComponent) component).scannerId(), toReportMeasure(measure)); } + public boolean hasIssues(DefaultInputComponent inputComponent) { + return reportPublisher.getReader().hasIssues(inputComponent.scannerId()); + } + public static ScannerReport.Measure toReportMeasure(DefaultMeasure measureToSave) { ScannerReport.Measure.Builder builder = ScannerReport.Measure.newBuilder(); builder.setMetricKey(measureToSave.metric().key()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java index 86c54c41d69..a573f94ff0e 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java @@ -27,7 +27,6 @@ import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.sensor.cache.ReadCache; import org.sonar.api.batch.sensor.cache.WriteCache; -import org.sonar.api.batch.sensor.internal.SensorStorage; import org.sonar.api.config.Configuration; import org.sonar.api.config.Settings; import org.sonar.scanner.cache.AnalysisCacheEnabled; @@ -39,7 +38,7 @@ public class ModuleSensorContext extends ProjectSensorContext { private final InputModule module; public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, Settings mutableModuleSettings, FileSystem fs, ActiveRules activeRules, - SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, + DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache, AnalysisCacheEnabled analysisCacheEnabled) { super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled); this.module = module; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java index a4cd85157d6..4c72730b94c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java @@ -40,7 +40,6 @@ import org.sonar.api.batch.sensor.cpd.internal.DefaultCpdTokens; import org.sonar.api.batch.sensor.error.NewAnalysisError; import org.sonar.api.batch.sensor.highlighting.NewHighlighting; import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting; -import org.sonar.api.batch.sensor.internal.SensorStorage; import org.sonar.api.batch.sensor.issue.NewExternalIssue; import org.sonar.api.batch.sensor.issue.NewIssue; import org.sonar.api.batch.sensor.issue.internal.DefaultExternalIssue; @@ -67,7 +66,7 @@ public class ProjectSensorContext implements SensorContext { private final Settings mutableSettings; private final FileSystem fs; private final ActiveRules activeRules; - private final SensorStorage sensorStorage; + private final DefaultSensorStorage sensorStorage; private final DefaultInputProject project; private final SonarRuntime sonarRuntime; private final Configuration config; @@ -77,7 +76,7 @@ public class ProjectSensorContext implements SensorContext { private final AnalysisCacheEnabled analysisCacheEnabled; public ProjectSensorContext(DefaultInputProject project, Configuration config, Settings mutableSettings, FileSystem fs, ActiveRules activeRules, - SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache, + DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache, AnalysisCacheEnabled analysisCacheEnabled) { this.project = project; this.config = config; @@ -194,6 +193,12 @@ public class ProjectSensorContext implements SensorContext { } @Override + public void markAsUnchanged(InputFile inputFile) { + DefaultInputFile defaultInputFile = (DefaultInputFile) inputFile; + defaultInputFile.setMarkedAsUnchanged(true); + } + + @Override public WriteCache nextCache() { return writeCache; } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java index cc440601a2f..ae2f877d384 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java @@ -24,6 +24,7 @@ import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -40,6 +41,7 @@ import org.sonar.api.utils.log.LoggerLevel; import org.sonar.scanner.bootstrap.DefaultScannerWsClient; import org.sonar.scanner.bootstrap.GlobalAnalysisMode; import org.sonar.scanner.fs.InputModuleHierarchy; +import org.sonar.scanner.protocol.output.ScannerReport; import org.sonar.scanner.scan.ScanProperties; import org.sonar.scanner.scan.branch.BranchConfiguration; import org.sonarqube.ws.Ce; @@ -93,6 +95,14 @@ public class ReportPublisherTest { } @Test + public void checks_if_component_has_issues() { + underTest.getWriter().writeComponentIssues(1, List.of(ScannerReport.Issue.newBuilder().build())); + + assertThat(underTest.getReader().hasIssues(1)).isTrue(); + assertThat(underTest.getReader().hasIssues(2)).isFalse(); + } + + @Test public void use_30s_write_timeout() { MockWsResponse submitMockResponse = new MockWsResponse(); submitMockResponse.setContent(Ce.SubmitResponse.newBuilder().setTaskId("task-1234").build().toByteArray()); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java index bb5d8f78bea..877e2e3e8ac 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java @@ -205,6 +205,16 @@ public class DefaultSensorStorageTest { } @Test + public void has_issues_delegates_to_report_publisher() { + DefaultInputFile file1 = new TestInputFileBuilder("foo", "src/Foo1.php").setStatus(InputFile.Status.SAME).build(); + DefaultInputFile file2 = new TestInputFileBuilder("foo", "src/Foo2.php").setStatus(InputFile.Status.SAME).build(); + + reportWriter.writeComponentIssues(file1.scannerId(), List.of(ScannerReport.Issue.newBuilder().build())); + assertThat(underTest.hasIssues(file1)).isTrue(); + assertThat(underTest.hasIssues(file2)).isFalse(); + } + + @Test public void should_save_highlighting() { DefaultInputFile file = new TestInputFileBuilder("foo", "src/Foo.php") .setContents("// comment").build(); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java index d40c7344bee..fb719656262 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java @@ -32,7 +32,6 @@ import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.measure.MetricFinder; import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.rule.internal.ActiveRulesBuilder; -import org.sonar.api.batch.sensor.internal.SensorStorage; import org.sonar.api.config.internal.MapSettings; import org.sonar.api.internal.SonarRuntimeImpl; import org.sonar.api.measures.CoreMetrics; @@ -55,7 +54,7 @@ public class ModuleSensorContextTest { private DefaultFileSystem fs; private ModuleSensorContext adaptor; private MapSettings settings; - private SensorStorage sensorStorage; + private DefaultSensorStorage sensorStorage; private SonarRuntime runtime; private BranchConfiguration branchConfiguration; private WriteCacheImpl writeCache; @@ -70,7 +69,7 @@ public class ModuleSensorContextTest { when(metricFinder.<Integer>findByKey(CoreMetrics.NCLOC_KEY)).thenReturn(CoreMetrics.NCLOC); when(metricFinder.<String>findByKey(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY)).thenReturn(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION); settings = new MapSettings(); - sensorStorage = mock(SensorStorage.class); + sensorStorage = mock(DefaultSensorStorage.class); branchConfiguration = mock(BranchConfiguration.class); writeCache = mock(WriteCacheImpl.class); readCache = mock(ReadCacheImpl.class); diff --git a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java index 006bdf66760..b4ba603055e 100644 --- a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java +++ b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java @@ -108,6 +108,11 @@ public class ScannerReportReader { return emptyCloseableIterator(); } + public boolean hasIssues(int componentRef) { + File file = fileStructure.fileFor(FileStructure.Domain.ISSUES, componentRef); + return fileExists(file); + } + public CloseableIterator<ScannerReport.ExternalIssue> readComponentExternalIssues(int componentRef) { File file = fileStructure.fileFor(FileStructure.Domain.EXTERNAL_ISSUES, componentRef); if (fileExists(file)) { diff --git a/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto b/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto index 9b2b159dc31..4870c034282 100644 --- a/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto +++ b/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto @@ -131,6 +131,7 @@ message Component { // Path relative to project base directory string project_relative_path = 14; + bool markedAsUnchanged = 15; enum ComponentType { UNSET = 0; |