SONAR-8622 Lazily generate metadata for input files

11 files changed, 287 insertions, 232 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AdditionalFilePredicates.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AdditionalFilePredicates.java
index 77e1bc39827..713ca7a3d89 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AdditionalFilePredicates.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AdditionalFilePredicates.java
@@ -19,9 +19,8 @@
  */
 package org.sonar.scanner.scan.filesystem;
 
-import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.IndexedFile;
 import org.sonar.api.batch.fs.internal.AbstractFilePredicate;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
 
 /**
  * Additional {@link org.sonar.api.batch.fs.FilePredicate}s that are
@@ -41,8 +40,8 @@ class AdditionalFilePredicates {
     }
 
     @Override
-    public boolean apply(InputFile f) {
-      return key.equals(((DefaultInputFile) f).key());
+    public boolean apply(IndexedFile f) {
+      return key.equals(f.key());
     }
   }
 
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DefaultModuleFileSystem.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DefaultModuleFileSystem.java
index fc2c46cfb6f..03b9fb9285e 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DefaultModuleFileSystem.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DefaultModuleFileSystem.java
@@ -26,12 +26,12 @@ import java.nio.charset.Charset;
 import java.util.List;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.InputFile.Status;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Project;
 import org.sonar.api.utils.MessageException;
 import org.sonar.scanner.analysis.DefaultAnalysisMode;
+import org.sonar.scanner.repository.ProjectRepositories;
 
 /**
  * @since 3.5
@@ -46,22 +46,26 @@ public class DefaultModuleFileSystem extends DefaultFileSystem {
   private List<File> testDirsOrFiles = Lists.newArrayList();
   private ComponentIndexer componentIndexer;
   private boolean initialized;
+  private Charset charset = null;
 
   public DefaultModuleFileSystem(ModuleInputFileCache moduleInputFileCache, Project project,
-    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer, DefaultAnalysisMode mode) {
+    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer, DefaultAnalysisMode mode,
+    ProjectRepositories projectRepositories) {
     super(initializer.baseDir(), moduleInputFileCache);
-    setFields(project, settings, indexer, initializer, componentIndexer, mode);
+    setFields(project, settings, indexer, initializer, componentIndexer, mode, projectRepositories);
   }
 
   @VisibleForTesting
   public DefaultModuleFileSystem(Project project,
-    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer, DefaultAnalysisMode mode) {
+    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer, DefaultAnalysisMode mode,
+    ProjectRepositories projectRepositories) {
     super(initializer.baseDir().toPath());
-    setFields(project, settings, indexer, initializer, componentIndexer, mode);
+    setFields(project, settings, indexer, initializer, componentIndexer, mode, projectRepositories);
   }
 
   private void setFields(Project project,
-    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer, DefaultAnalysisMode mode) {
+    Settings settings, FileIndexer indexer, ModuleFileSystemInitializer initializer, ComponentIndexer componentIndexer, DefaultAnalysisMode mode,
+    ProjectRepositories projectRepositories) {
     this.componentIndexer = componentIndexer;
     this.moduleKey = project.getKey();
     this.settings = settings;
@@ -72,7 +76,7 @@ public class DefaultModuleFileSystem extends DefaultFileSystem {
 
     // filter the files sensors have access to
     if (!mode.scanAllFiles()) {
-      setDefaultPredicate(predicates.not(predicates.hasStatus(Status.SAME)));
+      setDefaultPredicate(new SameInputFilePredicate(projectRepositories, moduleKey));
     }
   }
 
@@ -94,12 +98,13 @@ public class DefaultModuleFileSystem extends DefaultFileSystem {
 
   @Override
   public Charset encoding() {
-    final Charset charset;
-    String encoding = settings.getString(CoreProperties.ENCODING_PROPERTY);
-    if (StringUtils.isNotEmpty(encoding)) {
-      charset = Charset.forName(StringUtils.trim(encoding));
-    } else {
-      charset = Charset.defaultCharset();
+    if (charset == null) {
+      String encoding = settings.getString(CoreProperties.ENCODING_PROPERTY);
+      if (StringUtils.isNotEmpty(encoding)) {
+        charset = Charset.forName(StringUtils.trim(encoding));
+      } else {
+        charset = Charset.defaultCharset();
+      }
     }
     return charset;
   }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ExclusionFilters.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ExclusionFilters.java
index 2d7975e7ea3..e0dfdd1c5dd 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ExclusionFilters.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ExclusionFilters.java
@@ -23,6 +23,7 @@ import org.apache.commons.lang.ArrayUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.batch.ScannerSide;
+import org.sonar.api.batch.fs.IndexedFile;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.PathPattern;
 import org.sonar.api.scan.filesystem.FileExclusions;
@@ -67,7 +68,7 @@ public class ExclusionFilters {
     }
   }
 
-  public boolean accept(InputFile inputFile, InputFile.Type type) {
+  public boolean accept(IndexedFile indexedFile, InputFile.Type type) {
     PathPattern[] inclusionPatterns;
     PathPattern[] exclusionPatterns;
     if (InputFile.Type.MAIN == type) {
@@ -83,7 +84,7 @@ public class ExclusionFilters {
     if (inclusionPatterns.length > 0) {
       boolean matchInclusion = false;
       for (PathPattern pattern : inclusionPatterns) {
-        matchInclusion |= pattern.match(inputFile);
+        matchInclusion |= pattern.match(indexedFile);
       }
       if (!matchInclusion) {
         return false;
@@ -91,7 +92,7 @@ public class ExclusionFilters {
     }
     if (exclusionPatterns.length > 0) {
       for (PathPattern pattern : exclusionPatterns) {
-        if (pattern.match(inputFile)) {
+        if (pattern.match(indexedFile)) {
           return false;
         }
       }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
index 13aa962ccca..595773f62b5 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
@@ -19,7 +19,6 @@
  */
 package org.sonar.scanner.scan.filesystem;
 
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.FileSystemLoopException;
@@ -30,26 +29,23 @@ import java.nio.file.Files;
 import java.nio.file.LinkOption;
 import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.batch.ScannerSide;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
+import org.sonar.api.batch.fs.IndexedFile;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.InputFile.Type;
-import org.sonar.api.batch.fs.InputFileFilter;
+import org.sonar.api.batch.fs.internal.DefaultIndexedFile;
 import org.sonar.api.batch.fs.internal.DefaultInputDir;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.scan.filesystem.PathResolver;
+import org.sonar.api.batch.fs.InputFileFilter;
 import org.sonar.api.utils.MessageException;
 import org.sonar.scanner.util.ProgressReport;
 
@@ -63,21 +59,22 @@ public class FileIndexer {
   private final InputFileFilter[] filters;
   private final boolean isAggregator;
   private final ExclusionFilters exclusionFilters;
-  private final InputFileBuilderFactory inputFileBuilderFactory;
 
   private ProgressReport progressReport;
-  private ExecutorService executorService;
-  private List<Future<Void>> tasks;
+  private IndexedFileBuilder indexedFileBuilder;
+  private MetadataGenerator metadataGenerator;
 
-  public FileIndexer(ExclusionFilters exclusionFilters, InputFileBuilderFactory inputFileBuilderFactory, ProjectDefinition def, InputFileFilter[] filters) {
+  public FileIndexer(ExclusionFilters exclusionFilters, IndexedFileBuilder indexedFileBuilder, MetadataGenerator inputFileBuilder, ProjectDefinition def,
+    InputFileFilter[] filters) {
+    this.indexedFileBuilder = indexedFileBuilder;
+    this.metadataGenerator = inputFileBuilder;
     this.filters = filters;
     this.exclusionFilters = exclusionFilters;
-    this.inputFileBuilderFactory = inputFileBuilderFactory;
     this.isAggregator = !def.getSubProjects().isEmpty();
   }
 
-  public FileIndexer(ExclusionFilters exclusionFilters, InputFileBuilderFactory inputFileBuilderFactory, ProjectDefinition def) {
-    this(exclusionFilters, inputFileBuilderFactory, def, new InputFileFilter[0]);
+  public FileIndexer(ExclusionFilters exclusionFilters, IndexedFileBuilder indexedFileBuilder, MetadataGenerator inputFileBuilder, ProjectDefinition def) {
+    this(exclusionFilters, indexedFileBuilder, inputFileBuilder, def, new InputFileFilter[0]);
   }
 
   void index(DefaultModuleFileSystem fileSystem) {
@@ -91,14 +88,8 @@ public class FileIndexer {
 
     Progress progress = new Progress();
 
-    InputFileBuilder inputFileBuilder = inputFileBuilderFactory.create(fileSystem);
-    int threads = Math.max(1, Runtime.getRuntime().availableProcessors() - 1);
-    executorService = Executors.newFixedThreadPool(threads, new ThreadFactoryBuilder().setNameFormat("FileIndexer-%d").build());
-    tasks = new ArrayList<>();
-    indexFiles(fileSystem, progress, inputFileBuilder, fileSystem.sources(), InputFile.Type.MAIN);
-    indexFiles(fileSystem, progress, inputFileBuilder, fileSystem.tests(), InputFile.Type.TEST);
-
-    waitForTasksToComplete();
+    indexFiles(fileSystem, progress, fileSystem.sources(), InputFile.Type.MAIN);
+    indexFiles(fileSystem, progress, fileSystem.tests(), InputFile.Type.TEST);
 
     progressReport.stop(progress.count() + " files indexed");
 
@@ -107,27 +98,13 @@ public class FileIndexer {
     }
   }
 
-  private void waitForTasksToComplete() {
-    executorService.shutdown();
-    for (Future<Void> task : tasks) {
-      try {
-        task.get();
-      } catch (ExecutionException e) {
-        // Unwrap ExecutionException
-        throw e.getCause() instanceof RuntimeException ? (RuntimeException) e.getCause() : new IllegalStateException(e.getCause());
-      } catch (InterruptedException e) {
-        throw new IllegalStateException(e);
-      }
-    }
-  }
-
-  private void indexFiles(DefaultModuleFileSystem fileSystem, Progress progress, InputFileBuilder inputFileBuilder, List<File> sources, InputFile.Type type) {
+  private void indexFiles(DefaultModuleFileSystem fileSystem, Progress progress, List<File> sources, InputFile.Type type) {
     try {
       for (File dirOrFile : sources) {
         if (dirOrFile.isDirectory()) {
-          indexDirectory(inputFileBuilder, fileSystem, progress, dirOrFile, type);
+          indexDirectory(fileSystem, progress, dirOrFile, type);
         } else {
-          indexFile(inputFileBuilder, fileSystem, progress, dirOrFile.toPath(), type);
+          indexFile(fileSystem, progress, dirOrFile.toPath(), type);
         }
       }
     } catch (IOException e) {
@@ -135,57 +112,45 @@ public class FileIndexer {
     }
   }
 
-  private void indexDirectory(final InputFileBuilder inputFileBuilder, final DefaultModuleFileSystem fileSystem, final Progress status,
-    final File dirToIndex, final InputFile.Type type) throws IOException {
+  private void indexDirectory(final DefaultModuleFileSystem fileSystem, final Progress status, final File dirToIndex, final InputFile.Type type) throws IOException {
     Files.walkFileTree(dirToIndex.toPath().normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
-      new IndexFileVisitor(inputFileBuilder, fileSystem, status, type));
+      new IndexFileVisitor(fileSystem, status, type));
   }
 
-  private void indexFile(InputFileBuilder inputFileBuilder, DefaultModuleFileSystem fileSystem, Progress progress, Path sourceFile, InputFile.Type type) throws IOException {
+  private void indexFile(DefaultModuleFileSystem fileSystem, Progress progress, Path sourceFile, InputFile.Type type) throws IOException {
     // get case of real file without resolving link
     Path realFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS);
-    DefaultInputFile inputFile = inputFileBuilder.create(realFile.toFile());
-    if (inputFile != null) {
-      // Set basedir on input file prior to adding it to the FS since exclusions filters may require the absolute path
-      inputFile.setModuleBaseDir(fileSystem.baseDirPath());
-      if (exclusionFilters.accept(inputFile, type)) {
-        indexFile(inputFileBuilder, fileSystem, progress, inputFile, type);
+    DefaultIndexedFile indexedFile = indexedFileBuilder.create(realFile, type, fileSystem.baseDirPath());
+    if (indexedFile != null) {
+      if (exclusionFilters.accept(indexedFile, type)) {
+        InputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.readMetadata(f, fileSystem.encoding()));
+        if (accept(inputFile)) {
+          fileSystem.add(inputFile);
+        }
+        indexParentDir(fileSystem, indexedFile);
+        progress.markAsIndexed(indexedFile);
+        LOG.debug("'{}' indexed {} with language '{}'", indexedFile.relativePath(), type == Type.TEST ? "as test " : "", indexedFile.language());
       } else {
         progress.increaseExcludedByPatternsCount();
       }
     }
   }
 
-  private void indexFile(final InputFileBuilder inputFileBuilder, final DefaultModuleFileSystem fs,
-    final Progress status, final DefaultInputFile inputFile, final InputFile.Type type) {
-
-    tasks.add(executorService.submit(() -> {
-      DefaultInputFile completedInputFile = inputFileBuilder.completeAndComputeMetadata(inputFile, type);
-      if (completedInputFile != null && accept(completedInputFile)) {
-        LOG.debug("'{}' indexed {}with language '{}' and charset '{}'",
-          inputFile.relativePath(),
-          type == Type.TEST ? "as test " : "",
-          inputFile.language(),
-          inputFile.charset());
-        fs.add(completedInputFile);
-        status.markAsIndexed(completedInputFile);
-        File parentDir = completedInputFile.file().getParentFile();
-        String relativePath = new PathResolver().relativePath(fs.baseDir(), parentDir);
-        if (relativePath != null) {
-          DefaultInputDir inputDir = new DefaultInputDir(fs.moduleKey(), relativePath);
-          fs.add(inputDir);
-        }
-      }
-      return null;
-    }));
-
+  private void indexParentDir(DefaultModuleFileSystem fileSystem, IndexedFile indexedFile) {
+    File parentDir = indexedFile.file().getParentFile();
+    String relativePath = new PathResolver().relativePath(fileSystem.baseDir(), parentDir);
+    if (relativePath != null) {
+      DefaultInputDir inputDir = new DefaultInputDir(fileSystem.moduleKey(), relativePath);
+      inputDir.setModuleBaseDir(fileSystem.baseDirPath());
+      fileSystem.add(inputDir);
+    }
   }
 
-  private boolean accept(InputFile inputFile) {
+  private boolean accept(InputFile indexedFile) {
     // InputFileFilter extensions
     for (InputFileFilter filter : filters) {
-      if (!filter.accept(inputFile)) {
-        LOG.debug("'{}' excluded by {}", inputFile.relativePath(), filter.getClass().getName());
+      if (!filter.accept(indexedFile)) {
+        LOG.debug("'{}' excluded by {}", indexedFile.relativePath(), filter.getClass().getName());
         return false;
       }
     }
@@ -193,13 +158,11 @@ public class FileIndexer {
   }
 
   private class IndexFileVisitor implements FileVisitor<Path> {
-    private InputFileBuilder inputFileBuilder;
     private DefaultModuleFileSystem fileSystem;
     private Progress status;
     private Type type;
 
-    IndexFileVisitor(InputFileBuilder inputFileBuilder, DefaultModuleFileSystem fileSystem, Progress status, InputFile.Type type) {
-      this.inputFileBuilder = inputFileBuilder;
+    IndexFileVisitor(DefaultModuleFileSystem fileSystem, Progress status, InputFile.Type type) {
       this.fileSystem = fileSystem;
       this.status = status;
       this.type = type;
@@ -221,7 +184,7 @@ public class FileIndexer {
     @Override
     public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
       if (!Files.isHidden(file)) {
-        indexFile(inputFileBuilder, fileSystem, status, file, type);
+        indexFile(fileSystem, status, file, type);
       }
       return FileVisitResult.CONTINUE;
     }
@@ -246,7 +209,7 @@ public class FileIndexer {
     private final Set<Path> indexed = new HashSet<>();
     private int excludedByPatternsCount = 0;
 
-    synchronized void markAsIndexed(InputFile inputFile) {
+    synchronized void markAsIndexed(IndexedFile inputFile) {
       if (indexed.contains(inputFile.path())) {
         throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce "
           + "disjoint sets for main and test files");
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/IndexedFileBuilder.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/IndexedFileBuilder.java
new file mode 100644
index 00000000000..352065dbcb7
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/IndexedFileBuilder.java
@@ -0,0 +1,65 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import java.nio.file.Path;
+
+import javax.annotation.CheckForNull;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultIndexedFile;
+import org.sonar.api.config.Settings;
+import org.sonar.api.scan.filesystem.PathResolver;
+
+public class IndexedFileBuilder {
+  private static final Logger LOG = LoggerFactory.getLogger(IndexedFileBuilder.class);
+  private final String moduleKey;
+  private final PathResolver pathResolver;
+  private final LanguageDetection langDetection;
+  private final Settings settings;
+
+  IndexedFileBuilder(String moduleKey, PathResolver pathResolver, Settings settings, LanguageDetection langDetection) {
+    this.moduleKey = moduleKey;
+    this.pathResolver = pathResolver;
+    this.settings = settings;
+    this.langDetection = langDetection;
+  }
+
+  @CheckForNull
+  DefaultIndexedFile create(Path file, InputFile.Type type, Path moduleBaseDir) {
+    String relativePath = pathResolver.relativePath(moduleBaseDir, file);
+    if (relativePath == null) {
+      LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", file.toAbsolutePath(), moduleBaseDir);
+      return null;
+    }
+    DefaultIndexedFile indexedFile = new DefaultIndexedFile(moduleKey, moduleBaseDir, relativePath, type);
+    String language = langDetection.language(indexedFile);
+    if (language == null && !settings.getBoolean(CoreProperties.IMPORT_UNKNOWN_FILES_KEY)) {
+      LOG.debug("'{}' language is not supported by any analyzer. Skipping it.", relativePath);
+      return null;
+    }
+
+    indexedFile.setLanguage(language);
+    return indexedFile;
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/IndexedFileBuilderProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/IndexedFileBuilderProvider.java
new file mode 100644
index 00000000000..c892bf9ff8f
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/IndexedFileBuilderProvider.java
@@ -0,0 +1,33 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import org.picocontainer.injectors.ProviderAdapter;
+import org.sonar.api.batch.bootstrap.ProjectDefinition;
+import org.sonar.api.config.Settings;
+import org.sonar.api.scan.filesystem.PathResolver;
+
+public class IndexedFileBuilderProvider extends ProviderAdapter {
+
+  public IndexedFileBuilder provide(ProjectDefinition def, PathResolver pathResolver, Settings settings, LanguageDetectionFactory langDetectionFactory) {
+    return new IndexedFileBuilder(def.getKeyWithBranch(), pathResolver, settings, langDetectionFactory.create());
+  }
+
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilderFactory.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilderFactory.java
deleted file mode 100644
index 310484591ab..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilderFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import org.sonar.api.batch.ScannerSide;
-import org.sonar.api.batch.bootstrap.ProjectDefinition;
-import org.sonar.api.batch.fs.internal.FileMetadata;
-import org.sonar.api.config.Settings;
-import org.sonar.api.scan.filesystem.PathResolver;
-
-@ScannerSide
-public class InputFileBuilderFactory {
-
-  private final String moduleKey;
-  private final PathResolver pathResolver;
-  private final LanguageDetectionFactory langDetectionFactory;
-  private final StatusDetectionFactory statusDetectionFactory;
-  private final Settings settings;
-  private final FileMetadata fileMetadata;
-
-  public InputFileBuilderFactory(ProjectDefinition def, PathResolver pathResolver, LanguageDetectionFactory langDetectionFactory,
-    StatusDetectionFactory statusDetectionFactory, Settings settings, FileMetadata fileMetadata) {
-    this.fileMetadata = fileMetadata;
-    this.moduleKey = def.getKeyWithBranch();
-    this.pathResolver = pathResolver;
-    this.langDetectionFactory = langDetectionFactory;
-    this.statusDetectionFactory = statusDetectionFactory;
-    this.settings = settings;
-  }
-
-  InputFileBuilder create(DefaultModuleFileSystem fs) {
-    return new InputFileBuilder(moduleKey, pathResolver, langDetectionFactory.create(), statusDetectionFactory.create(), fs, settings, fileMetadata);
-  }
-}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
index 10708b1d392..b83402a7733 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
@@ -30,7 +30,7 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultIndexedFile;
 import org.sonar.api.batch.fs.internal.PathPattern;
 import org.sonar.api.config.Settings;
 import org.sonar.api.utils.MessageException;
@@ -88,7 +88,7 @@ class LanguageDetection {
   }
 
   @CheckForNull
-  String language(InputFile inputFile) {
+  String language(DefaultIndexedFile inputFile) {
     String detectedLanguage = null;
     for (String languageKey : languagesToConsider) {
       if (isCandidateForLanguage(inputFile, languageKey)) {
@@ -113,7 +113,7 @@ class LanguageDetection {
     return null;
   }
 
-  private boolean isCandidateForLanguage(InputFile inputFile, String languageKey) {
+  private boolean isCandidateForLanguage(DefaultIndexedFile inputFile, String languageKey) {
     PathPattern[] patterns = patternsByLanguage.get(languageKey);
     if (patterns != null) {
       for (PathPattern pathPattern : patterns) {
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MetadataGenerator.java
index 0cdf970a4e1..5468bd7ae6e 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MetadataGenerator.java
@@ -20,107 +20,36 @@
 package org.sonar.scanner.scan.filesystem;
 
 import com.google.common.annotations.VisibleForTesting;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-import javax.annotation.CheckForNull;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.FileSystem;
-import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.InputFile.Type;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.fs.internal.FileMetadata;
-import org.sonar.api.config.Settings;
-import org.sonar.api.scan.filesystem.PathResolver;
-
-class InputFileBuilder {
-
-  private static final Logger LOG = LoggerFactory.getLogger(InputFileBuilder.class);
+import org.sonar.api.batch.fs.internal.Metadata;
 
+class MetadataGenerator {
+  private static final Logger LOG = LoggerFactory.getLogger(MetadataGenerator.class);
   @VisibleForTesting
   static final Charset UTF_32BE = Charset.forName("UTF-32BE");
 
   @VisibleForTesting
   static final Charset UTF_32LE = Charset.forName("UTF-32LE");
 
-  private final String moduleKey;
-  private final PathResolver pathResolver;
-  private final LanguageDetection langDetection;
   private final StatusDetection statusDetection;
-  private final DefaultModuleFileSystem fs;
-  private final Settings settings;
   private final FileMetadata fileMetadata;
 
-  InputFileBuilder(String moduleKey, PathResolver pathResolver, LanguageDetection langDetection,
-    StatusDetection statusDetection, DefaultModuleFileSystem fs, Settings settings, FileMetadata fileMetadata) {
-    this.moduleKey = moduleKey;
-    this.pathResolver = pathResolver;
-    this.langDetection = langDetection;
+  MetadataGenerator(StatusDetection statusDetection, FileMetadata fileMetadata) {
     this.statusDetection = statusDetection;
-    this.fs = fs;
-    this.settings = settings;
     this.fileMetadata = fileMetadata;
   }
 
-  String moduleKey() {
-    return moduleKey;
-  }
-
-  PathResolver pathResolver() {
-    return pathResolver;
-  }
-
-  LanguageDetection langDetection() {
-    return langDetection;
-  }
-
-  StatusDetection statusDetection() {
-    return statusDetection;
-  }
-
-  FileSystem fs() {
-    return fs;
-  }
-
-  @CheckForNull
-  DefaultInputFile create(File file) {
-    String relativePath = pathResolver.relativePath(fs.baseDir(), file);
-    if (relativePath == null) {
-      LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", file.getAbsolutePath(), fs.baseDir());
-      return null;
-    }
-    return new DefaultInputFile(moduleKey, relativePath);
-  }
-
-  /**
-   * Optimization to not compute InputFile metadata if the file is excluded from analysis.
-   */
-  @CheckForNull
-  DefaultInputFile completeAndComputeMetadata(DefaultInputFile inputFile, InputFile.Type type) {
-    inputFile.setType(type);
-    inputFile.setModuleBaseDir(fs.baseDir().toPath());
-
-    String lang = langDetection.language(inputFile);
-    if (lang == null && !settings.getBoolean(CoreProperties.IMPORT_UNKNOWN_FILES_KEY)) {
-      // Return fast to skip costly metadata computation
-      LOG.debug("'{}' language is not supported by any analyzer. Skipping it.", inputFile.relativePath());
-      return null;
-    }
-    inputFile.setLanguage(lang);
-
-    Charset charset = detectCharset(inputFile.file(), fs.encoding());
-    inputFile.setCharset(charset);
-
-    inputFile.initMetadata(fileMetadata.readMetadata(inputFile.file(), charset));
-
-    inputFile.setStatus(statusDetection.status(inputFile.moduleKey(), inputFile.relativePath(), inputFile.hash()));
-
-    return inputFile;
-  }
-
   /**
    * @return charset detected from BOM in given file or given defaultCharset
    * @throws IllegalStateException if an I/O error occurs
@@ -146,4 +75,18 @@ class InputFileBuilder {
       throw new IllegalStateException("Unable to read file " + file.getAbsolutePath(), e);
     }
   }
+
+  public Metadata readMetadata(final DefaultInputFile inputFile, Charset defaultEncoding) {
+    try {
+      Charset charset = detectCharset(inputFile.file(), defaultEncoding);
+      inputFile.setCharset(charset);
+      Metadata metadata = fileMetadata.readMetadata(inputFile.file(), charset);
+      inputFile.setStatus(statusDetection.status(inputFile.moduleKey(), inputFile.relativePath(), metadata.hash()));
+      LOG.debug("'{}' generated metadata {} with and charset '{}'",
+        inputFile.relativePath(), inputFile.type() == Type.TEST ? "as test " : "", charset);
+      return metadata;
+    } catch (Exception e) {
+      throw new IllegalStateException(e);
+    }
+  }
 }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MetadataGeneratorProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MetadataGeneratorProvider.java
new file mode 100644
index 00000000000..13bfd484184
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MetadataGeneratorProvider.java
@@ -0,0 +1,31 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import org.picocontainer.injectors.ProviderAdapter;
+import org.sonar.api.batch.ScannerSide;
+import org.sonar.api.batch.fs.internal.FileMetadata;
+
+@ScannerSide
+public class MetadataGeneratorProvider extends ProviderAdapter {
+  public MetadataGenerator provide(StatusDetectionFactory statusDetectionFactory, FileMetadata fileMetadata) {
+    return new MetadataGenerator(statusDetectionFactory.create(), fileMetadata);
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/SameInputFilePredicate.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/SameInputFilePredicate.java
new file mode 100644
index 00000000000..5ec413082cd
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/SameInputFilePredicate.java
@@ -0,0 +1,66 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.InputFilePredicate;
+import org.sonar.scanner.repository.FileData;
+import org.sonar.scanner.repository.ProjectRepositories;
+
+public class SameInputFilePredicate implements InputFilePredicate {
+  private static final Logger LOG = LoggerFactory.getLogger(SameInputFilePredicate.class);
+  private final ProjectRepositories projectRepositories;
+  private final String moduleKey;
+
+  public SameInputFilePredicate(ProjectRepositories projectRepositories, String moduleKey) {
+    this.projectRepositories = projectRepositories;
+    this.moduleKey = moduleKey;
+  }
+
+  @Override
+  public boolean apply(InputFile inputFile) {
+    FileData fileDataPerPath = projectRepositories.fileData(moduleKey, inputFile.relativePath());
+    if (fileDataPerPath == null) {
+      // ADDED
+      return true;
+    }
+    String previousHash = fileDataPerPath.hash();
+    if (StringUtils.isEmpty(previousHash)) {
+      // ADDED
+      return true;
+    }
+
+    // this will trigger computation of metadata
+    String hash = ((DefaultInputFile) inputFile).hash();
+    if (StringUtils.equals(hash, previousHash)) {
+      // SAME
+      LOG.debug("'{}' filtering unmodified file", inputFile.relativePath());
+      return false;
+    }
+
+    // CHANGED
+    return true;
+  }
+
+}