SONAR-6931 Speed up issues mode by scanning only changed files

6 files changed, 96 insertions, 68 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/DefaultProjectTree.java b/sonar-batch/src/main/java/org/sonar/batch/DefaultProjectTree.java
index b520c1bcf17..674a5db2182 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/DefaultProjectTree.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/DefaultProjectTree.java
@@ -33,7 +33,7 @@ import org.sonar.batch.scan.ImmutableProjectReactor;
 public class DefaultProjectTree implements Startable {
 
   private final ProjectConfigurator configurator;
-  private ImmutableProjectReactor projectReactor;
+  private final ImmutableProjectReactor projectReactor;
 
   private List<Project> projects;
   private Map<ProjectDefinition, Project> projectsByDef;
diff --git a/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java b/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java
index b4d2e3f322e..2e0d79cf9d1 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java
@@ -36,9 +36,11 @@ import java.util.Map;
 public class DefaultAnalysisMode extends AbstractAnalysisMode implements AnalysisMode {
 
   private static final Logger LOG = LoggerFactory.getLogger(DefaultAnalysisMode.class);
+  private static final String KEY_ONLY_ANALYZE_CHANGED = "sonar.scanChangedFilesOnly";
 
   private boolean mediumTestMode;
   private boolean notAssociated;
+  private boolean onlyChanged;
 
   public DefaultAnalysisMode(GlobalProperties globalProps, AnalysisProperties props) {
     init(globalProps.properties(), props.properties());
@@ -52,6 +54,10 @@ public class DefaultAnalysisMode extends AbstractAnalysisMode implements Analysi
     return notAssociated;
   }
 
+  public boolean onlyAnalyzeChanged() {
+    return onlyChanged;
+  }
+
   private void init(Map<String, String> globalProps, Map<String, String> analysisProps) {
     // make sure analysis is consistent with global properties
     boolean globalPreview = isIssues(globalProps);
@@ -70,6 +76,8 @@ public class DefaultAnalysisMode extends AbstractAnalysisMode implements Analysi
     issues = CoreProperties.ANALYSIS_MODE_ISSUES.equals(mode) || CoreProperties.ANALYSIS_MODE_PREVIEW.equals(mode);
     mediumTestMode = "true".equals(getPropertyWithFallback(analysisProps, globalProps, FakePluginInstaller.MEDIUM_TEST_ENABLED));
     notAssociated = issues && rootProjectKeyMissing(analysisProps);
+    String onlyChangedStr = getPropertyWithFallback(analysisProps, globalProps, KEY_ONLY_ANALYZE_CHANGED);
+    onlyChanged = issues && "true".equals(onlyChangedStr);
   }
 
   public void printMode() {
@@ -86,6 +94,9 @@ public class DefaultAnalysisMode extends AbstractAnalysisMode implements Analysi
     if (notAssociated) {
       LOG.info("Local analysis");
     }
+    if (onlyChanged) {
+      LOG.info("Scanning only changed files");
+    }
   }
 
   private static String getPropertyWithFallback(Map<String, String> props1, Map<String, String> props2, String key) {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java
index 5273748a7f4..eaaf466228e 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java
@@ -19,14 +19,20 @@
  */
 package org.sonar.batch.issue.tracking;
 
+import org.sonar.api.batch.fs.InputFile.Status;
+
+import org.sonar.batch.analysis.DefaultAnalysisMode;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
 import java.util.List;
 import java.util.Set;
+
 import javax.annotation.CheckForNull;
+
 import org.sonar.api.batch.BatchSide;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.rule.ActiveRule;
@@ -54,15 +60,18 @@ public class LocalIssueTracking {
   private final ActiveRules activeRules;
   private final ServerIssueRepository serverIssueRepository;
   private final Date analysisDate;
+  private final DefaultAnalysisMode mode;
 
   private boolean hasServerAnalysis;
 
   public LocalIssueTracking(BatchComponentCache resourceCache, IssueTracking tracking, ServerLineHashesLoader lastLineHashes, IssueUpdater updater,
-    ActiveRules activeRules, ServerIssueRepository serverIssueRepository, ProjectRepositories projectRepositories, ReportPublisher reportPublisher) {
+    ActiveRules activeRules, ServerIssueRepository serverIssueRepository, ProjectRepositories projectRepositories, ReportPublisher reportPublisher,
+    DefaultAnalysisMode mode) {
     this.tracking = tracking;
     this.lastLineHashes = lastLineHashes;
     this.updater = updater;
     this.serverIssueRepository = serverIssueRepository;
+    this.mode = mode;
     this.analysisDate = ((Project) resourceCache.getRoot().resource()).getAnalysisDate();
     this.changeContext = IssueChangeContext.createScan(analysisDate);
     this.activeRules = activeRules;
@@ -78,18 +87,23 @@ public class LocalIssueTracking {
   public List<DefaultIssue> trackIssues(BatchComponent component, Set<BatchReport.Issue> rawIssues) {
     List<DefaultIssue> trackedIssues = Lists.newArrayList();
     if (hasServerAnalysis) {
-
       // all the issues that are not closed in db before starting this module scan, including manual issues
       Collection<ServerIssue> serverIssues = loadServerIssues(component);
 
-      SourceHashHolder sourceHashHolder = loadSourceHashes(component);
+      if (shouldCopyServerIssues(component)) {
+        // raw issues should be empty, we just need to deal with server issues (SONAR-6931)
+        copyServerIssues(serverIssues, trackedIssues);
+      } else {
 
-      IssueTrackingResult trackingResult = tracking.track(sourceHashHolder, serverIssues, rawIssues);
+        SourceHashHolder sourceHashHolder = loadSourceHashes(component);
 
-      // unmatched from server = issues that have been resolved + issues on disabled/removed rules + manual issues
-      addUnmatchedFromServer(trackingResult.unmatched(), sourceHashHolder, trackedIssues);
+        IssueTrackingResult trackingResult = tracking.track(sourceHashHolder, serverIssues, rawIssues);
 
-      mergeMatched(component, trackingResult, trackedIssues, rawIssues);
+        // unmatched from server = issues that have been resolved + issues on disabled/removed rules + manual issues
+        addUnmatchedFromServer(trackingResult.unmatched(), sourceHashHolder, trackedIssues);
+
+        mergeMatched(component, trackingResult, trackedIssues, rawIssues);
+      }
     }
 
     if (hasServerAnalysis && ResourceUtils.isRootProject(component.resource())) {
@@ -100,8 +114,33 @@ public class LocalIssueTracking {
     return trackedIssues;
   }
 
+  private boolean shouldCopyServerIssues(BatchComponent component) {
+    if (mode.onlyAnalyzeChanged() && component.isFile()) {
+      DefaultInputFile inputFile = (DefaultInputFile) component.inputComponent();
+      if (inputFile.status() == Status.SAME) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private void copyServerIssues(Collection<ServerIssue> serverIssues, List<DefaultIssue> trackedIssues) {
+    for (ServerIssue serverIssue : serverIssues) {
+      org.sonar.batch.protocol.input.BatchInput.ServerIssue unmatchedPreviousIssue = ((ServerIssueFromWs) serverIssue).getDto();
+      DefaultIssue unmatched = toUnmatchedIssue(unmatchedPreviousIssue);
+
+      ActiveRule activeRule = activeRules.find(unmatched.ruleKey());
+      unmatched.setNew(false);
+
+      boolean isRemovedRule = activeRule == null;
+      unmatched.setBeingClosed(isRemovedRule);
+      unmatched.setOnDisabledRule(isRemovedRule);
+      trackedIssues.add(unmatched);
+    }
+  }
+
   private DefaultIssue toTracked(BatchComponent component, BatchReport.Issue rawIssue) {
-    DefaultIssue trackedIssue = new org.sonar.core.issue.DefaultIssueBuilder()
+    return new org.sonar.core.issue.DefaultIssueBuilder()
       .componentKey(component.key())
       .projectKey("unused")
       .ruleKey(RuleKey.of(rawIssue.getRuleRepository(), rawIssue.getRuleKey()))
@@ -110,7 +149,6 @@ public class LocalIssueTracking {
       .message(rawIssue.hasMsg() ? rawIssue.getMsg() : null)
       .severity(rawIssue.getSeverity().name())
       .build();
-    return trackedIssue;
   }
 
   @CheckForNull
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
index e4e5a1602ee..29488db579d 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
@@ -53,7 +53,7 @@ public class ComponentIndexer {
   public void execute(DefaultModuleFileSystem fs) {
     module.setBaseDir(fs.baseDir());
 
-    for (InputFile inputFile : fs.inputFiles(fs.predicates().all())) {
+    for (InputFile inputFile : fs.inputFiles()) {
       String languageKey = inputFile.language();
       boolean unitTest = InputFile.Type.TEST == inputFile.type();
       Resource sonarFile = File.create(inputFile.relativePath(), languages.get(languageKey), unitTest);
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
index 24e26f5eb94..17d6f8d4271 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
@@ -19,19 +19,25 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import org.sonar.api.batch.fs.InputFile.Status;
+
+import org.sonar.batch.analysis.DefaultAnalysisMode;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.collect.Collections2;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+
 import java.io.File;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
+
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.fs.FilePredicate;
@@ -59,7 +65,7 @@ public class DefaultModuleFileSystem extends DefaultFileSystem implements Module
   private boolean initialized;
 
   public DefaultModuleFileSystem(ModuleInputFileCache moduleInputFileCache, Project project,
-    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer) {
+    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer, DefaultAnalysisMode mode) {
     super(initializer.baseDir(), moduleInputFileCache);
     this.componentIndexer = componentIndexer;
     this.moduleKey = project.getKey();
@@ -70,11 +76,16 @@ public class DefaultModuleFileSystem extends DefaultFileSystem implements Module
     this.sourceDirsOrFiles = initializer.sources();
     this.testDirsOrFiles = initializer.tests();
     this.binaryDirs = initializer.binaryDirs();
+
+    // filter the files that sensors have access to (SONAR-6931)
+    if (mode.onlyAnalyzeChanged()) {
+      setDefaultPredicate(predicates.not(predicates.hasStatus(Status.SAME)));
+    }
   }
 
   @VisibleForTesting
   public DefaultModuleFileSystem(Project project,
-    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer) {
+    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer, DefaultAnalysisMode mode) {
     super(initializer.baseDir().toPath());
     this.componentIndexer = componentIndexer;
     this.moduleKey = project.getKey();
@@ -85,6 +96,11 @@ public class DefaultModuleFileSystem extends DefaultFileSystem implements Module
     this.sourceDirsOrFiles = initializer.sources();
     this.testDirsOrFiles = initializer.tests();
     this.binaryDirs = initializer.binaryDirs();
+    
+    // filter the files sensors have access to
+    if (mode.onlyAnalyzeChanged()) {
+      setDefaultPredicate(predicates.not(predicates.hasStatus(Status.SAME)));
+    }
   }
 
   public boolean isInitialized() {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputPathCache.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputPathCache.java
index 3164a69d10a..195f1441bb4 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputPathCache.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputPathCache.java
@@ -19,19 +19,14 @@
  */
 package org.sonar.batch.scan.filesystem;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import javax.annotation.CheckForNull;
+import com.google.common.collect.Table;
+import com.google.common.collect.TreeBasedTable;
 import org.sonar.api.batch.BatchSide;
 import org.sonar.api.batch.fs.InputDir;
 import org.sonar.api.batch.fs.InputFile;
 
+import javax.annotation.CheckForNull;
+
 /**
  * Cache of all files and dirs. This cache is shared amongst all project modules. Inclusion and
  * exclusion patterns are already applied.
@@ -39,91 +34,59 @@ import org.sonar.api.batch.fs.InputFile;
 @BatchSide
 public class InputPathCache {
 
-  private final Map<String, SortedMap<String, InputFile>> inputFileCache = new LinkedHashMap<>();
-  private final Map<String, SortedMap<String, InputDir>> inputDirCache = new LinkedHashMap<>();
+  private final Table<String, String, InputFile> inputFileCache = TreeBasedTable.create();
+  private final Table<String, String, InputDir> inputDirCache = TreeBasedTable.create();
 
   public Iterable<InputFile> allFiles() {
-    return Iterables.concat(Iterables.transform(inputFileCache.values(), new Function<Map<String, InputFile>, Collection<InputFile>>() {
-      @Override
-      public Collection<InputFile> apply(Map<String, InputFile> input) {
-        return input.values();
-      }
-    }));
+    return inputFileCache.values();
   }
 
   public Iterable<InputDir> allDirs() {
-    return Iterables.concat(Iterables.transform(inputDirCache.values(), new Function<Map<String, InputDir>, Collection<InputDir>>() {
-      @Override
-      public Collection<InputDir> apply(Map<String, InputDir> input) {
-        return input.values();
-      }
-    }));
+    return inputDirCache.values();
   }
 
   public Iterable<InputFile> filesByModule(String moduleKey) {
-    if (inputFileCache.containsKey(moduleKey)) {
-      return inputFileCache.get(moduleKey).values();
-    }
-    return Collections.emptyList();
+    return inputFileCache.row(moduleKey).values();
   }
 
   public Iterable<InputDir> dirsByModule(String moduleKey) {
-    if (inputDirCache.containsKey(moduleKey)) {
-      return inputDirCache.get(moduleKey).values();
-    }
-    return Collections.emptyList();
+    return inputDirCache.row(moduleKey).values();
   }
 
   public InputPathCache removeModule(String moduleKey) {
-    inputFileCache.remove(moduleKey);
-    inputDirCache.remove(moduleKey);
+    inputFileCache.row(moduleKey).clear();
+    inputDirCache.row(moduleKey).clear();
     return this;
   }
 
   public InputPathCache remove(String moduleKey, InputFile inputFile) {
-    if (inputFileCache.containsKey(moduleKey)) {
-      inputFileCache.get(moduleKey).remove(inputFile.relativePath());
-    }
+    inputFileCache.remove(moduleKey, inputFile.relativePath());
     return this;
   }
 
   public InputPathCache remove(String moduleKey, InputDir inputDir) {
-    if (inputDirCache.containsKey(moduleKey)) {
-      inputDirCache.get(moduleKey).remove(inputDir.relativePath());
-    }
+    inputDirCache.remove(moduleKey, inputDir.relativePath());
     return this;
   }
 
   public InputPathCache put(String moduleKey, InputFile inputFile) {
-    if (!inputFileCache.containsKey(moduleKey)) {
-      inputFileCache.put(moduleKey, new TreeMap<String, InputFile>());
-    }
-    inputFileCache.get(moduleKey).put(inputFile.relativePath(), inputFile);
+    inputFileCache.put(moduleKey, inputFile.relativePath(), inputFile);
     return this;
   }
 
   public InputPathCache put(String moduleKey, InputDir inputDir) {
-    if (!inputDirCache.containsKey(moduleKey)) {
-      inputDirCache.put(moduleKey, new TreeMap<String, InputDir>());
-    }
-    inputDirCache.get(moduleKey).put(inputDir.relativePath(), inputDir);
+    inputDirCache.put(moduleKey, inputDir.relativePath(), inputDir);
     return this;
   }
 
   @CheckForNull
   public InputFile getFile(String moduleKey, String relativePath) {
-    if (inputFileCache.containsKey(moduleKey)) {
-      return inputFileCache.get(moduleKey).get(relativePath);
-    }
-    return null;
+    return inputFileCache.get(moduleKey, relativePath);
   }
 
   @CheckForNull
   public InputDir getDir(String moduleKey, String relativePath) {
-    if (inputDirCache.containsKey(moduleKey)) {
-      return inputDirCache.get(moduleKey).get(relativePath);
-    }
-    return null;
+    return inputDirCache.get(moduleKey, relativePath);
   }
 
 }