SONAR-11621 Use scm based exclusion in scanner engine

2 files changed, 85 insertions, 32 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
index 0cc8ea42b64..8e4f75b0205 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
@@ -23,8 +23,8 @@ import java.io.IOException;
 import java.nio.file.LinkOption;
 import java.nio.file.Path;
 import java.util.Arrays;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BooleanSupplier;
+import javax.annotation.Nullable;
 import org.apache.commons.io.FilenameUtils;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.fs.InputFile;
@@ -35,6 +35,7 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputModule;
 import org.sonar.api.batch.fs.internal.DefaultInputProject;
 import org.sonar.api.batch.fs.internal.SensorStrategy;
+import org.sonar.api.batch.scm.IgnoreCommand;
 import org.sonar.api.notifications.AnalysisWarnings;
 import org.sonar.api.utils.MessageException;
 import org.sonar.api.utils.log.Logger;
@@ -89,16 +90,11 @@ public class FileIndexer {
     ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader,
     MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties) {
     this(project, scannerComponentIdGenerator, componentStore, projectExclusionFilters, projectCoverageAndDuplicationExclusions, issueExclusionsLoader, metadataGenerator,
-      sensorStrategy,
-      languageDetection,
-      analysisWarnings,
-      properties, new InputFileFilter[0]);
+      sensorStrategy, languageDetection, analysisWarnings, properties, new InputFileFilter[0]);
   }
 
   void indexFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
-    Path sourceFile,
-    InputFile.Type type, ProgressReport progressReport,
-    AtomicInteger excludedByPatternsCount)
+    Path sourceFile, Type type, ProgressReport progressReport, ProjectFileIndexer.ExclusionCounter exclusionCounter, @Nullable IgnoreCommand ignoreCommand)
     throws IOException {
     // get case of real file without resolving link
     Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
@@ -114,12 +110,12 @@ public class FileIndexer {
     Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile);
     boolean included = evaluateInclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
     if (!included) {
-      excludedByPatternsCount.incrementAndGet();
+      exclusionCounter.increaseByPatternsCount();
       return;
     }
     boolean excluded = evaluateExclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
     if (excluded) {
-      excludedByPatternsCount.incrementAndGet();
+      exclusionCounter.increaseByPatternsCount();
       return;
     }
     String language = langDetection.language(realAbsoluteFile, projectRelativePath);
@@ -127,6 +123,13 @@ public class FileIndexer {
       LOG.warn("File '{}' is ignored because it doesn't belong to the forced language '{}'", realAbsoluteFile.toAbsolutePath(), langDetection.getForcedLanguage());
       return;
     }
+
+    if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) {
+      LOG.debug("File '{}' is excluded by the scm ignore settings.");
+      exclusionCounter.increaseByScmCount();
+      return;
+    }
+
     DefaultIndexedFile indexedFile = new DefaultIndexedFile(realAbsoluteFile, project.key(),
       projectRelativePath.toString(),
       moduleRelativePath.toString(),
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java
index 9c13d0c2e1d..85dd31eb3ec 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java
@@ -41,6 +41,7 @@ import org.apache.commons.lang.SystemUtils;
 import org.sonar.api.batch.fs.InputFile.Type;
 import org.sonar.api.batch.fs.internal.DefaultInputModule;
 import org.sonar.api.batch.fs.internal.InputModuleHierarchy;
+import org.sonar.api.batch.scm.IgnoreCommand;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
@@ -49,6 +50,7 @@ import org.sonar.scanner.bootstrap.GlobalServerSettings;
 import org.sonar.scanner.scan.ModuleConfiguration;
 import org.sonar.scanner.scan.ModuleConfigurationProvider;
 import org.sonar.scanner.scan.ProjectServerSettings;
+import org.sonar.scanner.scm.ScmConfiguration;
 import org.sonar.scanner.util.ProgressReport;
 
 /**
@@ -59,18 +61,21 @@ public class ProjectFileIndexer {
   private static final Logger LOG = Loggers.get(ProjectFileIndexer.class);
   private final ProjectExclusionFilters projectExclusionFilters;
   private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions;
+  private ScmConfiguration scmConfiguration;
   private final InputComponentStore componentStore;
   private final InputModuleHierarchy inputModuleHierarchy;
   private final GlobalConfiguration globalConfig;
   private final GlobalServerSettings globalServerSettings;
   private final ProjectServerSettings projectServerSettings;
   private final FileIndexer fileIndexer;
+  private final IgnoreCommand ignoreCommand;
+  private final boolean useScmExclusion;
 
   private ProgressReport progressReport;
 
   public ProjectFileIndexer(InputComponentStore componentStore, ProjectExclusionFilters exclusionFilters,
     InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings,
-    FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions) {
+    FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, ScmConfiguration scmConfiguration) {
     this.componentStore = componentStore;
     this.inputModuleHierarchy = inputModuleHierarchy;
     this.globalConfig = globalConfig;
@@ -79,6 +84,9 @@ public class ProjectFileIndexer {
     this.fileIndexer = fileIndexer;
     this.projectExclusionFilters = exclusionFilters;
     this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions;
+    this.scmConfiguration = scmConfiguration;
+    this.ignoreCommand = loadIgnoreCommand();
+    this.useScmExclusion = ignoreCommand != null;
   }
 
   public void index() {
@@ -87,26 +95,47 @@ public class ProjectFileIndexer {
     LOG.info("Project configuration:");
     projectExclusionFilters.log("  ");
     projectCoverageAndDuplicationExclusions.log("  ");
+    ExclusionCounter exclusionCounter = new ExclusionCounter();
 
-    AtomicInteger excludedByPatternsCount = new AtomicInteger(0);
-
-    indexModulesRecursively(inputModuleHierarchy.root(), excludedByPatternsCount);
+    if (useScmExclusion) {
+      ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath());
+      indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
+      ignoreCommand.clean();
+    } else {
+      indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
+    }
 
     int totalIndexed = componentStore.inputFiles().size();
     progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed");
 
-    int excludedFileCount = excludedByPatternsCount.get();
-    if (projectExclusionFilters.hasPattern() || excludedFileCount > 0) {
-      LOG.info("{} {} ignored because of inclusion/exclusion patterns", excludedFileCount, pluralizeFiles(excludedFileCount));
+    int excludedFileByPatternCount = exclusionCounter.getByPatternsCount();
+    if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) {
+      LOG.info("{} {} ignored because of inclusion/exclusion patterns", excludedFileByPatternCount, pluralizeFiles(excludedFileByPatternCount));
+    }
+    int excludedFileByScmCount = exclusionCounter.getByScmCount();
+    if (useScmExclusion) {
+      LOG.info("{} {} ignored because of scm ignore settings", excludedFileByScmCount, pluralizeFiles(excludedFileByScmCount));
+    }
+  }
+
+  private IgnoreCommand loadIgnoreCommand() {
+    try {
+      if (!scmConfiguration.isExclusionDisabled() && scmConfiguration.provider() != null) {
+        return scmConfiguration.provider().ignoreCommand();
+      }
+    } catch (UnsupportedOperationException e) {
+      LOG.debug("File exclusion based on SCM ignore information is not available with this plugin.");
     }
+
+    return null;
   }
 
-  private void indexModulesRecursively(DefaultInputModule module, AtomicInteger excludedByPatternsCount) {
-    inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(m -> indexModulesRecursively(m, excludedByPatternsCount));
-    index(module, excludedByPatternsCount);
+  private void indexModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) {
+    inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(m -> indexModulesRecursively(m, exclusionCounter));
+    index(module, exclusionCounter);
   }
 
-  private void index(DefaultInputModule module, AtomicInteger excludedByPatternsCount) {
+  private void index(DefaultInputModule module, ExclusionCounter exclusionCounter) {
     // Emulate creation of module level settings
     ModuleConfiguration moduleConfig = new ModuleConfigurationProvider().provide(globalConfig, module, globalServerSettings, projectServerSettings);
     ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig);
@@ -119,8 +148,8 @@ public class ProjectFileIndexer {
       moduleExclusionFilters.log("  ");
       moduleCoverageAndDuplicationExclusions.log("  ");
     }
-    indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, module.getSourceDirsOrFiles(), Type.MAIN, excludedByPatternsCount);
-    indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, module.getTestDirsOrFiles(), Type.TEST, excludedByPatternsCount);
+    indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, module.getSourceDirsOrFiles(), Type.MAIN, exclusionCounter);
+    indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, module.getTestDirsOrFiles(), Type.TEST, exclusionCounter);
   }
 
   private static void logPaths(String label, Path baseDir, List<Path> paths) {
@@ -153,13 +182,13 @@ public class ProjectFileIndexer {
   }
 
   private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
-    List<Path> sources, Type type, AtomicInteger excludedByPatternsCount) {
+    List<Path> sources, Type type, ExclusionCounter exclusionCounter) {
     try {
       for (Path dirOrFile : sources) {
         if (dirOrFile.toFile().isDirectory()) {
-          indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, excludedByPatternsCount);
+          indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, exclusionCounter);
         } else {
-          fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, excludedByPatternsCount);
+          fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, exclusionCounter, ignoreCommand);
         }
       }
     } catch (IOException e) {
@@ -168,10 +197,10 @@ public class ProjectFileIndexer {
   }
 
   private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters,
-    ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type, AtomicInteger excludedByPatternsCount)
+    ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type, ExclusionCounter exclusionCounter)
     throws IOException {
     Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
-      new IndexFileVisitor(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, type, excludedByPatternsCount));
+      new IndexFileVisitor(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, type, exclusionCounter));
   }
 
   private class IndexFileVisitor implements FileVisitor<Path> {
@@ -179,16 +208,16 @@ public class ProjectFileIndexer {
     private final ModuleExclusionFilters moduleExclusionFilters;
     private final ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions;
     private final Type type;
-    private final AtomicInteger excludedByPatternsCount;
+    private final ExclusionCounter exclusionCounter;
 
     IndexFileVisitor(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
       Type type,
-      AtomicInteger excludedByPatternsCount) {
+      ExclusionCounter exclusionCounter) {
       this.module = module;
       this.moduleExclusionFilters = moduleExclusionFilters;
       this.moduleCoverageAndDuplicationExclusions = moduleCoverageAndDuplicationExclusions;
       this.type = type;
-      this.excludedByPatternsCount = excludedByPatternsCount;
+      this.exclusionCounter = exclusionCounter;
     }
 
     @Override
@@ -202,7 +231,7 @@ public class ProjectFileIndexer {
     @Override
     public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
       if (!Files.isHidden(file)) {
-        fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, file, type, progressReport, excludedByPatternsCount);
+        fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, file, type, progressReport, exclusionCounter, ignoreCommand);
       }
       return FileVisitResult.CONTINUE;
     }
@@ -235,4 +264,25 @@ public class ProjectFileIndexer {
       }
     }
   }
+
+  static class ExclusionCounter {
+    private final AtomicInteger excludedByPatternsCount = new AtomicInteger(0);
+    private final AtomicInteger excludedByScmCount = new AtomicInteger(0);
+
+    public void increaseByPatternsCount() {
+      excludedByPatternsCount.incrementAndGet();
+    }
+
+    public int getByPatternsCount() {
+      return excludedByScmCount.get();
+    }
+
+    public void increaseByScmCount() {
+      excludedByPatternsCount.incrementAndGet();
+    }
+
+    public int getByScmCount() {
+      return excludedByScmCount.get();
+    }
+  }
 }