SONAR-11367 Write duplication for changed files in PR and SLB

2 files changed, 56 insertions, 25 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/CpdExecutor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/CpdExecutor.java
index 878cf807324..464241ab670 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/CpdExecutor.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/CpdExecutor.java
@@ -20,9 +20,11 @@
 package org.sonar.scanner.cpd;
 
 import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -34,6 +36,7 @@ import java.util.stream.Collectors;
 import org.sonar.api.batch.fs.InputComponent;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputComponent;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.duplications.block.Block;
@@ -66,15 +69,21 @@ public class CpdExecutor {
   private final InputComponentStore componentStore;
   private final ProgressReport progressReport;
   private final CpdSettings settings;
-  private int count;
+  private final ExecutorService executorService;
+  private int count = 0;
   private int total;
 
   public CpdExecutor(CpdSettings settings, SonarCpdBlockIndex index, ReportPublisher publisher, InputComponentStore inputComponentCache) {
+    this(settings, index, publisher, inputComponentCache, Executors.newSingleThreadExecutor());
+  }
+
+  public CpdExecutor(CpdSettings settings, SonarCpdBlockIndex index, ReportPublisher publisher, InputComponentStore inputComponentCache, ExecutorService executorService) {
     this.settings = settings;
     this.index = index;
     this.publisher = publisher;
     this.componentStore = inputComponentCache;
     this.progressReport = new ProgressReport("CPD computation", TimeUnit.SECONDS.toMillis(10));
+    this.executorService = executorService;
   }
 
   public void execute() {
@@ -83,19 +92,28 @@ public class CpdExecutor {
 
   @VisibleForTesting
   void execute(long timeout) {
-    total = index.noResources();
-    int filesWithoutBlocks = index.noIndexedFiles() - total;
+    List<FileBlocks> components = new ArrayList<>(index.noResources());
+    Iterator<ResourceBlocks> it = index.iterator();
+
+    while (it.hasNext()) {
+      ResourceBlocks resourceBlocks = it.next();
+      Optional<FileBlocks> fileBlocks = toFileBlocks(resourceBlocks.resourceId(), resourceBlocks.blocks());
+      if (!fileBlocks.isPresent() || fileBlocks.get().getInputFile().status() == InputFile.Status.SAME) {
+        continue;
+      }
+      components.add(fileBlocks.get());
+    }
+
+    int filesWithoutBlocks = index.noIndexedFiles() - index.noResources();
     if (filesWithoutBlocks > 0) {
       LOG.info("{} {} had no CPD blocks", filesWithoutBlocks, pluralize(filesWithoutBlocks));
     }
+
+    total = components.size();
     progressReport.start(String.format("Calculating CPD for %d %s", total, pluralize(total)));
-    ExecutorService executorService = Executors.newSingleThreadExecutor();
     try {
-      Iterator<ResourceBlocks> it = index.iterator();
-
-      while (it.hasNext()) {
-        ResourceBlocks resourceBlocks = it.next();
-        runCpdAnalysis(executorService, resourceBlocks.resourceId(), resourceBlocks.blocks(), timeout);
+      for (FileBlocks fileBlocks : components) {
+        runCpdAnalysis(executorService, fileBlocks.getInputFile(), fileBlocks.getBlocks(), timeout);
         count++;
       }
       progressReport.stop("CPD calculation finished");
@@ -112,18 +130,7 @@ public class CpdExecutor {
   }
 
   @VisibleForTesting
-  void runCpdAnalysis(ExecutorService executorService, String componentKey, final Collection<Block> fileBlocks, long timeout) {
-    DefaultInputComponent component = (DefaultInputComponent) componentStore.getByKey(componentKey);
-    if (component == null) {
-      LOG.error("Resource not found in component store: {}. Skipping CPD computation for it", componentKey);
-      return;
-    }
-
-    InputFile inputFile = (InputFile) component;
-    if (inputFile.status() == InputFile.Status.SAME) {
-      return;
-    }
-
+  void runCpdAnalysis(ExecutorService executorService, DefaultInputFile inputFile, Collection<Block> fileBlocks, long timeout) {
     LOG.debug("Detection of duplications for {}", inputFile.absolutePath());
     progressReport.message(String.format("%d/%d - current file: %s", count, total, inputFile.absolutePath()));
 
@@ -150,7 +157,7 @@ public class CpdExecutor {
       filtered = duplications;
     }
 
-    saveDuplications(component, filtered);
+    saveDuplications(inputFile, filtered);
   }
 
   @VisibleForTesting final void saveDuplications(final DefaultInputComponent component, List<CloneGroup> duplications) {
@@ -173,6 +180,15 @@ public class CpdExecutor {
     publisher.getWriter().writeComponentDuplications(component.batchId(), reportDuplications);
   }
 
+  private Optional<FileBlocks> toFileBlocks(String componentKey, Collection<Block> fileBlocks) {
+    DefaultInputFile component = (DefaultInputFile) componentStore.getByKey(componentKey);
+    if (component == null) {
+      LOG.error("Resource not found in component store: {}. Skipping CPD computation for it", componentKey);
+      return Optional.empty();
+    }
+    return Optional.of(new FileBlocks(component, fileBlocks));
+  }
+
   private Duplication toReportDuplication(InputComponent component, Duplication.Builder dupBuilder, Duplicate.Builder blockBuilder, CloneGroup input) {
     dupBuilder.clear();
     ClonePart originBlock = input.getOriginPart();
@@ -207,4 +223,22 @@ public class CpdExecutor {
     }
     return dupBuilder.build();
   }
+
+  private static class FileBlocks {
+    public FileBlocks(DefaultInputFile inputFile, Collection<Block> blocks) {
+      this.inputFile = inputFile;
+      this.blocks = blocks;
+    }
+
+    private DefaultInputFile inputFile;
+    private Collection<Block> blocks;
+
+    public DefaultInputFile getInputFile() {
+      return inputFile;
+    }
+
+    public Collection<Block> getBlocks() {
+      return blocks;
+    }
+  }
 }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java
index 88f47c292ab..b410ed19c31 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java
@@ -506,9 +506,6 @@ public class DefaultSensorStorage implements SensorStorage {
   @Override
   public void store(DefaultCpdTokens defaultCpdTokens) {
     DefaultInputFile inputFile = (DefaultInputFile) defaultCpdTokens.inputFile();
-    if (shouldSkipStorage(inputFile)) {
-      return;
-    }
     inputFile.setPublished(true);
     PmdBlockChunker blockChunker = new PmdBlockChunker(getCpdBlockSize(inputFile.language()));
     List<Block> blocks = blockChunker.chunk(inputFile.key(), defaultCpdTokens.getTokenLines());