SONAR-11135 SONAR-11136 Load changed lines from SCM plugins and write in the scanner report

7 files changed, 291 insertions, 21 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ChangedLinesPublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ChangedLinesPublisher.java
new file mode 100644
index 00000000000..ef6dcc60278
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ChangedLinesPublisher.java
@@ -0,0 +1,101 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.report;
+
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.InputModuleHierarchy;
+import org.sonar.api.batch.scm.ScmProvider;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.api.utils.log.Profiler;
+import org.sonar.scanner.protocol.output.ScannerReport;
+import org.sonar.scanner.protocol.output.ScannerReportWriter;
+import org.sonar.scanner.scan.branch.BranchConfiguration;
+import org.sonar.scanner.scan.filesystem.InputComponentStore;
+import org.sonar.scanner.scm.ScmConfiguration;
+import org.sonar.scanner.util.ScannerUtils;
+
+public class ChangedLinesPublisher implements ReportPublisherStep {
+  private static final Logger LOG = Loggers.get(ChangedLinesPublisher.class);
+  private static final String LOG_MSG = "SCM writing changed lines";
+
+  private final ScmConfiguration scmConfiguration;
+  private final InputModuleHierarchy inputModuleHierarchy;
+  private final InputComponentStore inputComponentStore;
+  private final BranchConfiguration branchConfiguration;
+
+  public ChangedLinesPublisher(ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy, InputComponentStore inputComponentStore,
+    BranchConfiguration branchConfiguration) {
+    this.scmConfiguration = scmConfiguration;
+    this.inputModuleHierarchy = inputModuleHierarchy;
+    this.inputComponentStore = inputComponentStore;
+    this.branchConfiguration = branchConfiguration;
+  }
+
+  @Override public void publish(ScannerReportWriter writer) {
+    if (scmConfiguration.isDisabled() || !branchConfiguration.isShortOrPullRequest() || branchConfiguration.branchTarget() == null) {
+      return;
+    }
+
+    ScmProvider provider = scmConfiguration.provider();
+    if (provider == null) {
+      return;
+    }
+
+    Profiler profiler = Profiler.create(LOG).startInfo(LOG_MSG);
+    int count = writeChangedLines(provider, writer);
+    LOG.debug("SCM reported changed lines for {} {} in the branch", count, ScannerUtils.pluralize("file", count));
+    profiler.stopInfo();
+  }
+
+  private int writeChangedLines(ScmProvider provider, ScannerReportWriter writer) {
+    Path rootBaseDir = inputModuleHierarchy.root().getBaseDir();
+    Map<Path, DefaultInputFile> allPublishedFiles = StreamSupport.stream(inputComponentStore.allFilesToPublish().spliterator(), false)
+      .collect(Collectors.toMap(DefaultInputFile::path, f -> f));
+    Map<Path, Set<Integer>> pathSetMap = provider.branchChangedLines(branchConfiguration.branchTarget(), rootBaseDir, allPublishedFiles.keySet());
+    int count = 0;
+
+    if (pathSetMap == null) {
+      return count;
+    }
+
+    for (Map.Entry<Path, DefaultInputFile> e : allPublishedFiles.entrySet()) {
+      Set<Integer> changedLines = pathSetMap.get(e.getKey());
+      // if the file got no information returned by the SCM, we write nothing in the report and
+      // the compute engine will use SCM dates to estimate which lines are new
+      if (changedLines != null) {
+        count++;
+        writeChangedLines(writer, e.getValue().batchId(), changedLines);
+      }
+    }
+    return count;
+  }
+
+  private static void writeChangedLines(ScannerReportWriter writer, int fileRef, Set<Integer> changedLines) {
+    ScannerReport.ChangedLines.Builder builder = ScannerReport.ChangedLines.newBuilder();
+    builder.addAllLine(changedLines);
+    writer.writeComponentChangedLines(fileRef, builder.build());
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java
index c36b8a196e6..8ec6f7af82a 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ComponentsPublisher.java
@@ -69,7 +69,7 @@ public class ComponentsPublisher implements ReportPublisherStep {
   public void publish(ScannerReportWriter writer) {
     this.reader = new ScannerReportReader(writer.getFileStructure().root());
     this.writer = writer;
-    recursiveWriteComponent((DefaultInputComponent) moduleHierarchy.root());
+    recursiveWriteComponent(moduleHierarchy.root());
   }
 
   /**
@@ -152,8 +152,7 @@ public class ComponentsPublisher implements ReportPublisherStep {
   }
 
   private boolean shouldSkipComponent(DefaultInputComponent component, Collection<InputComponent> children) {
-    if (component instanceof InputModule && children.isEmpty()
-      && (branchConfiguration.isShortOrPullRequest())) {
+    if (component instanceof InputModule && children.isEmpty() && (branchConfiguration.isShortOrPullRequest())) {
       // no children on a module in short branch analysis -> skip it (except root)
       return !moduleHierarchy.isRoot((InputModule) component);
     } else if (component instanceof InputDir && children.isEmpty()) {
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java
index 0bb31674dd4..b2c105c2722 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java
@@ -57,6 +57,7 @@ import org.sonar.scanner.issue.tracking.ServerLineHashesLoader;
 import org.sonar.scanner.mediumtest.ScanTaskObservers;
 import org.sonar.scanner.report.ActiveRulesPublisher;
 import org.sonar.scanner.report.AnalysisContextReportPublisher;
+import org.sonar.scanner.report.ChangedLinesPublisher;
 import org.sonar.scanner.report.ComponentsPublisher;
 import org.sonar.scanner.report.ContextPropertiesPublisher;
 import org.sonar.scanner.report.CoveragePublisher;
@@ -118,9 +119,13 @@ public class ProjectScanContainer extends ComponentContainer {
     ProjectLock lock = getComponentByType(ProjectLock.class);
     lock.tryLock();
     getComponentByType(WorkDirectoriesInitializer.class).execute();
-    if (isTherePreviousAnalysis()) {
+
+    if (!isIssuesMode()) {
+      addReportPublishSteps();
+    } else if (isTherePreviousAnalysis()) {
       addIssueTrackingComponents();
     }
+
   }
 
   private void addBatchComponents() {
@@ -196,11 +201,6 @@ public class ProjectScanContainer extends ComponentContainer {
       AnalysisContextReportPublisher.class,
       MetadataPublisher.class,
       ActiveRulesPublisher.class,
-      ComponentsPublisher.class,
-      MeasuresPublisher.class,
-      CoveragePublisher.class,
-      SourcePublisher.class,
-      TestExecutionAndCoveragePublisher.class,
 
       // Cpd
       CpdExecutor.class,
@@ -216,6 +216,17 @@ public class ProjectScanContainer extends ComponentContainer {
     addIfMissing(DefaultProjectRepositoriesLoader.class, ProjectRepositoriesLoader.class);
   }
 
+  private void addReportPublishSteps() {
+    add(
+      ComponentsPublisher.class,
+      MeasuresPublisher.class,
+      CoveragePublisher.class,
+      SourcePublisher.class,
+      ChangedLinesPublisher.class,
+      TestExecutionAndCoveragePublisher.class
+    );
+  }
+
   private void addIssueTrackingComponents() {
     add(
       LocalIssueTracking.class,
@@ -228,6 +239,10 @@ public class ProjectScanContainer extends ComponentContainer {
     return getComponentByType(ProjectRepositories.class).lastAnalysisDate() != null;
   }
 
+  private boolean isIssuesMode() {
+    return getComponentByType(GlobalAnalysisMode.class).isIssues();
+  }
+
   private void addBatchExtensions() {
     getComponentByType(CoreExtensionsInstaller.class)
       .install(this, noExtensionFilter(), extension -> isInstantiationStrategy(extension, PER_BATCH));
@@ -250,7 +265,7 @@ public class ProjectScanContainer extends ComponentContainer {
     LOG.info("Project key: {}", tree.root().key());
     LOG.info("Project base dir: {}", tree.root().getBaseDir());
     properties.organizationKey().ifPresent(k -> LOG.info("Organization key: {}", k));
-    
+
     String branch = tree.root().definition().getBranch();
     if (branch != null) {
       LOG.info("Branch key: {}", branch);
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmChangedFilesProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmChangedFilesProvider.java
index a04b4f5021a..af13e28575c 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmChangedFilesProvider.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmChangedFilesProvider.java
@@ -30,6 +30,7 @@ import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.api.utils.log.Profiler;
 import org.sonar.scanner.scan.branch.BranchConfiguration;
+import org.sonar.scanner.util.ScannerUtils;
 
 public class ScmChangedFilesProvider extends ProviderAdapter {
   private static final Logger LOG = Loggers.get(ScmChangedFilesProvider.class);
@@ -69,7 +70,7 @@ public class ScmChangedFilesProvider extends ProviderAdapter {
         Collection<Path> changedFiles = scmProvider.branchChangedFiles(branchConfiguration.branchTarget(), rootBaseDir);
         profiler.stopInfo();
         if (changedFiles != null) {
-          LOG.debug("SCM reported {} {} changed in the branch", changedFiles.size(), pluralize("file", changedFiles.size()));
+          LOG.debug("SCM reported {} {} changed in the branch", changedFiles.size(), ScannerUtils.pluralize("file", changedFiles.size()));
           return changedFiles;
         }
       }
@@ -79,11 +80,4 @@ public class ScmChangedFilesProvider extends ProviderAdapter {
     return null;
   }
 
-  private static String pluralize(String str, int i) {
-    if (i == 1) {
-      return str;
-    }
-    return str + "s";
-  }
-
 }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmPublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmPublisher.java
index bb587269fd9..cd852fce2ef 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmPublisher.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmPublisher.java
@@ -29,6 +29,7 @@ import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.InputFile.Status;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.api.batch.scm.ScmProvider;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.scanner.protocol.output.ScannerReport;
@@ -71,18 +72,20 @@ public final class ScmPublisher {
       LOG.info("SCM Publisher is disabled");
       return;
     }
-    if (configuration.provider() == null) {
+
+    ScmProvider provider = configuration.provider();
+    if (provider == null) {
       LOG.info("No SCM system was detected. You can use the '" + CoreProperties.SCM_PROVIDER_KEY + "' property to explicitly specify it.");
       return;
     }
 
     List<InputFile> filesToBlame = collectFilesToBlame(writer);
     if (!filesToBlame.isEmpty()) {
-      String key = configuration.provider().key();
+      String key = provider.key();
       LOG.info("SCM provider for this project is: " + key);
       DefaultBlameOutput output = new DefaultBlameOutput(writer, filesToBlame);
       try {
-        configuration.provider().blameCommand().blame(new DefaultBlameInput(fs, filesToBlame), output);
+        provider.blameCommand().blame(new DefaultBlameInput(fs, filesToBlame), output);
       } catch (Exception e) {
         output.finish(false);
         throw e;
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/util/ScannerUtils.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/util/ScannerUtils.java
index 3358904661a..5fa46f186ea 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/util/ScannerUtils.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/util/ScannerUtils.java
@@ -63,4 +63,11 @@ public class ScannerUtils {
     return o.getClass().getName();
   }
 
+  public static String pluralize(String str, int i) {
+    if (i == 1) {
+      return str;
+    }
+    return str + "s";
+  }
+
 }
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ChangedLinesPublisherTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ChangedLinesPublisherTest.java
new file mode 100644
index 00000000000..78413d95a07
--- /dev/null
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ChangedLinesPublisherTest.java
@@ -0,0 +1,151 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.report;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.api.batch.fs.internal.InputModuleHierarchy;
+import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
+import org.sonar.api.batch.scm.ScmProvider;
+import org.sonar.scanner.protocol.output.ScannerReportReader;
+import org.sonar.scanner.protocol.output.ScannerReportWriter;
+import org.sonar.scanner.scan.branch.BranchConfiguration;
+import org.sonar.scanner.scan.filesystem.InputComponentStore;
+import org.sonar.scanner.scm.ScmConfiguration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+public class ChangedLinesPublisherTest {
+  private static final String TARGET_BRANCH = "target";
+  private static final Path BASE_DIR = Paths.get("/root");
+
+  private ScmConfiguration scmConfiguration = mock(ScmConfiguration.class);
+  private InputModuleHierarchy inputModuleHierarchy = mock(InputModuleHierarchy.class);
+  private InputComponentStore inputComponentStore = mock(InputComponentStore.class);
+  private BranchConfiguration branchConfiguration = mock(BranchConfiguration.class);
+  private ScannerReportWriter writer;
+  private ScmProvider provider = mock(ScmProvider.class);
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  private ChangedLinesPublisher publisher = new ChangedLinesPublisher(scmConfiguration, inputModuleHierarchy, inputComponentStore, branchConfiguration);
+
+  @Before
+  public void setUp() {
+    writer = new ScannerReportWriter(temp.getRoot());
+    when(branchConfiguration.isShortOrPullRequest()).thenReturn(true);
+    when(scmConfiguration.isDisabled()).thenReturn(false);
+    when(scmConfiguration.provider()).thenReturn(provider);
+    when(branchConfiguration.branchTarget()).thenReturn(TARGET_BRANCH);
+    DefaultInputModule root = mock(DefaultInputModule.class);
+    when(root.getBaseDir()).thenReturn(BASE_DIR);
+    when(inputModuleHierarchy.root()).thenReturn(root);
+  }
+
+  @Test
+  public void skip_if_scm_is_disabled() {
+    when(scmConfiguration.isDisabled()).thenReturn(true);
+    publisher.publish(writer);
+    verifyZeroInteractions(inputComponentStore, inputModuleHierarchy, provider);
+    assertNotPublished();
+  }
+
+  @Test
+  public void skip_if_not_pr_or_slb() {
+    when(branchConfiguration.isShortOrPullRequest()).thenReturn(false);
+    publisher.publish(writer);
+    verifyZeroInteractions(inputComponentStore, inputModuleHierarchy, provider);
+    assertNotPublished();
+  }
+
+  @Test
+  public void skip_if_target_branch_is_null() {
+    when(branchConfiguration.branchTarget()).thenReturn(null);
+    publisher.publish(writer);
+    verifyZeroInteractions(inputComponentStore, inputModuleHierarchy, provider);
+    assertNotPublished();
+  }
+
+  @Test
+  public void skip_if_no_scm_provider() {
+    when(scmConfiguration.provider()).thenReturn(null);
+    publisher.publish(writer);
+    verifyZeroInteractions(inputComponentStore, inputModuleHierarchy, provider);
+    assertNotPublished();
+  }
+
+  @Test
+  public void skip_if_scm_provider_returns_null() {
+    publisher.publish(writer);
+    assertNotPublished();
+  }
+
+  @Test
+  public void write_changed_files() {
+    DefaultInputFile fileWithChangedLines = createInputFile("path1");
+    DefaultInputFile fileWithoutChangedLines = createInputFile("path2");
+    Set<Path> paths = new HashSet<>(Arrays.asList(BASE_DIR.resolve("path1"), BASE_DIR.resolve("path2")));
+    Set<Integer> lines = new HashSet<>(Arrays.asList(1, 10));
+    when(provider.branchChangedLines(TARGET_BRANCH, BASE_DIR, paths)).thenReturn(Collections.singletonMap(BASE_DIR.resolve("path1"), lines));
+    when(inputComponentStore.allFilesToPublish()).thenReturn(Arrays.asList(fileWithChangedLines, fileWithoutChangedLines));
+
+    publisher.publish(writer);
+
+    assertPublished(fileWithChangedLines, lines);
+    assertNotPublished(fileWithoutChangedLines);
+  }
+
+  private DefaultInputFile createInputFile(String path) {
+    return new TestInputFileBuilder("module", path)
+      .setProjectBaseDir(BASE_DIR)
+      .setModuleBaseDir(BASE_DIR)
+      .build();
+  }
+
+  private void assertPublished(DefaultInputFile file, Set<Integer> lines) {
+    assertThat(new File(temp.getRoot(), "changed-lines-" + file.batchId() + ".pb")).exists();
+    ScannerReportReader reader = new ScannerReportReader(temp.getRoot());
+    assertThat(reader.readComponentChangedLines(file.batchId()).getLineList()).containsExactlyElementsOf(lines);
+  }
+
+  private void assertNotPublished(DefaultInputFile file) {
+    assertThat(new File(temp.getRoot(), "changed-lines-" + file.batchId() + ".pb")).doesNotExist();
+  }
+
+  private void assertNotPublished() {
+    assertThat(temp.getRoot().list()).isEmpty();
+  }
+
+}