SONAR-926 improve computation of number of lines

sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java

@@ -28,12 +28,7 @@ import org.sonar.api.batch.rule.CheckFactory;
 import org.sonar.api.platform.ComponentContainer;
 import org.sonar.api.resources.Project;
 import org.sonar.api.scan.filesystem.FileExclusions;
-import org.sonar.batch.DefaultProjectClasspath;
-import org.sonar.batch.DefaultSensorContext;
-import org.sonar.batch.DefaultTimeMachine;
-import org.sonar.batch.ProjectTree;
-import org.sonar.batch.ResourceFilters;
-import org.sonar.batch.ViolationFilters;
+import org.sonar.batch.*;
 import org.sonar.batch.bootstrap.BatchExtensionDictionnary;
 import org.sonar.batch.bootstrap.ExtensionInstaller;
 import org.sonar.batch.bootstrap.ExtensionMatcher;
@@ -51,17 +46,7 @@ import org.sonar.batch.rule.ModuleQProfiles;
 import org.sonar.batch.rule.ModuleRulesProvider;
 import org.sonar.batch.rule.QProfileSensor;
 import org.sonar.batch.rule.RulesProfileProvider;
-import org.sonar.batch.scan.filesystem.ComponentIndexer;
-import org.sonar.batch.scan.filesystem.DefaultModuleFileSystem;
-import org.sonar.batch.scan.filesystem.DeprecatedFileFilters;
-import org.sonar.batch.scan.filesystem.ExclusionFilters;
-import org.sonar.batch.scan.filesystem.FileHashes;
-import org.sonar.batch.scan.filesystem.FileIndex;
-import org.sonar.batch.scan.filesystem.FileSystemLogger;
-import org.sonar.batch.scan.filesystem.LanguageRecognizer;
-import org.sonar.batch.scan.filesystem.ModuleFileSystemInitializer;
-import org.sonar.batch.scan.filesystem.ProjectFileSystemAdapter;
-import org.sonar.batch.scan.filesystem.RemoteFileHashes;
+import org.sonar.batch.scan.filesystem.*;
 import org.sonar.batch.scan.language.DefaultModuleLanguages;
 import org.sonar.batch.scan.report.ComponentSelectorFactory;
 import org.sonar.batch.scan.report.JsonReport;
@@ -107,12 +92,13 @@ public class ModuleScanContainer extends ComponentContainer {
       FileExclusions.class,
       ExclusionFilters.class,
       DeprecatedFileFilters.class,
-      FileHashes.class,
-      RemoteFileHashes.class,
+      InputFileBuilderFactory.class,
+      StatusDetectionFactory.class,
+      LanguageDetectionFactory.class,
+      PreviousFileHashLoader.class,
       FileIndex.class,
       ComponentIndexer.class,
       DefaultModuleLanguages.class,
-      LanguageRecognizer.class,
       FileSystemLogger.class,
       DefaultProjectClasspath.class,
       DefaultModuleFileSystem.class,

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndex.java

@@ -19,37 +19,27 @@
  */
 package org.sonar.batch.scan.filesystem;
 
-import com.google.common.collect.Maps;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.apache.commons.io.filefilter.HiddenFileFilter;
 import org.apache.commons.io.filefilter.IOFileFilter;
-import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.BatchComponent;
-import org.sonar.api.resources.Java;
-import org.sonar.api.resources.JavaFile;
 import org.sonar.api.resources.Project;
 import org.sonar.api.scan.filesystem.InputDir;
 import org.sonar.api.scan.filesystem.InputFile;
-import org.sonar.api.scan.filesystem.InputFileFilter;
-import org.sonar.api.scan.filesystem.ModuleFileSystem;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.api.scan.filesystem.internal.DefaultInputDir;
-import org.sonar.api.scan.filesystem.internal.DefaultInputFile;
+import org.sonar.api.scan.filesystem.InputFileFilter;
 import org.sonar.api.utils.PathUtils;
 import org.sonar.api.utils.SonarException;
 
-import javax.annotation.CheckForNull;
-import javax.annotation.Nullable;
-
 import java.io.File;
-import java.nio.charset.Charset;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 /**
@@ -85,19 +75,17 @@ public class FileIndex implements BatchComponent {
 
   private final PathResolver pathResolver;
   private final List<InputFileFilter> filters;
-  private final LanguageRecognizer languageRecognizer;
   private final InputFileCache fileCache;
-  private final FileHashes fileHashes;
   private final Project module;
   private final ExclusionFilters exclusionFilters;
+  private final InputFileBuilderFactory inputFileBuilderFactory;
 
-  public FileIndex(List<InputFileFilter> filters, ExclusionFilters exclusionFilters, LanguageRecognizer languageRecognizer,
-    InputFileCache cache, FileHashes fileHashes, PathResolver pathResolver, Project project) {
+  public FileIndex(List<InputFileFilter> filters, ExclusionFilters exclusionFilters, InputFileBuilderFactory inputFileBuilderFactory,
+                   InputFileCache cache, PathResolver pathResolver, Project project) {
     this.filters = filters;
     this.exclusionFilters = exclusionFilters;
-    this.languageRecognizer = languageRecognizer;
+    this.inputFileBuilderFactory = inputFileBuilderFactory;
     this.fileCache = cache;
-    this.fileHashes = fileHashes;
     this.pathResolver = pathResolver;
     this.module = project;
   }
@@ -114,13 +102,14 @@ public class FileIndex implements BatchComponent {
 
     Progress progress = new Progress(fileCache.fileRelativePaths(fileSystem.moduleKey()));
 
+    InputFileBuilder inputFileBuilder = inputFileBuilderFactory.create(fileSystem);
     if (!fileSystem.sourceFiles().isEmpty() || !fileSystem.testFiles().isEmpty()) {
       // Index only provided files
-      indexFiles(fileSystem, progress, fileSystem.sourceFiles(), InputFile.TYPE_MAIN);
-      indexFiles(fileSystem, progress, fileSystem.testFiles(), InputFile.TYPE_TEST);
+      indexFiles(inputFileBuilder, fileSystem, progress, fileSystem.sourceFiles(), InputFile.TYPE_MAIN);
+      indexFiles(inputFileBuilder, fileSystem, progress, fileSystem.testFiles(), InputFile.TYPE_TEST);
     } else if (fileSystem.baseDir() != null) {
       // index from basedir
-      indexDirectory(fileSystem, progress, fileSystem.baseDir());
+      indexDirectory(inputFileBuilder, fileSystem, progress, fileSystem.baseDir());
     }
 
     // Remove files that have been removed since previous indexation
@@ -132,16 +121,16 @@ public class FileIndex implements BatchComponent {
 
   }
 
-  private void indexFiles(DefaultModuleFileSystem fileSystem, Progress progress, List<File> sourceFiles, String type) {
+  private void indexFiles(InputFileBuilder inputFileBuilder, DefaultModuleFileSystem fileSystem, Progress progress, List<File> sourceFiles, String type) {
     for (File sourceFile : sourceFiles) {
       String path = pathResolver.relativePath(fileSystem.baseDir(), sourceFile);
       if (path == null) {
         LoggerFactory.getLogger(getClass()).warn(String.format(
           "File '%s' is not declared in module basedir %s", sourceFile.getAbsoluteFile(), fileSystem.baseDir()
-          ));
+        ));
       } else {
         if (exclusionFilters.accept(sourceFile, path, type)) {
-          indexFile(fileSystem, progress, sourceFile, path, type);
+          indexFile(inputFileBuilder, fileSystem, progress, sourceFile, path, type);
         }
       }
     }
@@ -158,37 +147,35 @@ public class FileIndex implements BatchComponent {
 
   InputDir inputDir(DefaultModuleFileSystem fileSystem, File ioFile) {
     String path = computeFilePath(fileSystem, ioFile);
-    // TODO no cache for InputDir
-    Map<String, String> attributes = Maps.newHashMap();
-    // paths
+    DefaultInputDir inputDir = new DefaultInputDir(FilenameUtils.normalize(ioFile.getAbsolutePath(), true), path);
     String resourceKey = PathUtils.sanitize(path);
-    set(attributes, DefaultInputFile.ATTRIBUTE_COMPONENT_KEY, module.getEffectiveKey() + ":" + resourceKey);
-    return DefaultInputDir.create(ioFile, path, attributes);
+    inputDir.setKey(module.getEffectiveKey() + ":" + resourceKey);
+    return inputDir;
   }
 
-  private void indexDirectory(DefaultModuleFileSystem fileSystem, Progress status, File dirToIndex) {
+  private void indexDirectory(InputFileBuilder inputFileBuilder, DefaultModuleFileSystem fileSystem, Progress status, File dirToIndex) {
     Collection<File> files = FileUtils.listFiles(dirToIndex, FILE_FILTER, DIR_FILTER);
     for (File sourceFile : files) {
       String path = pathResolver.relativePath(fileSystem.baseDir(), sourceFile);
       if (path == null) {
         LoggerFactory.getLogger(getClass()).warn(String.format(
           "File '%s' is not declared in module basedir %s", sourceFile.getAbsoluteFile(), fileSystem.baseDir()
-          ));
+        ));
       } else {
         if (exclusionFilters.accept(sourceFile, path, InputFile.TYPE_MAIN)) {
-          indexFile(fileSystem, status, sourceFile, path, InputFile.TYPE_MAIN);
+          indexFile(inputFileBuilder, fileSystem, status, sourceFile, path, InputFile.TYPE_MAIN);
         }
         if (exclusionFilters.accept(sourceFile, path, InputFile.TYPE_TEST)) {
-          indexFile(fileSystem, status, sourceFile, path, InputFile.TYPE_TEST);
+          indexFile(inputFileBuilder, fileSystem, status, sourceFile, path, InputFile.TYPE_TEST);
         }
       }
     }
   }
 
-  private void indexFile(DefaultModuleFileSystem fileSystem, Progress status, File file, String path, String type) {
-    InputFile input = newInputFile(fileSystem, type, file, path);
-    if (input != null && accept(input)) {
-      fileCache.put(fileSystem.moduleKey(), input);
+  private void indexFile(InputFileBuilder inputFileBuilder, DefaultModuleFileSystem fs, Progress status, File file, String path, String type) {
+    InputFile inputFile = inputFileBuilder.create(file, type);
+    if (inputFile != null && accept(inputFile)) {
+      fileCache.put(fs.moduleKey(), inputFile);
       status.markAsIndexed(path);
     }
   }
@@ -197,67 +184,6 @@ public class FileIndex implements BatchComponent {
     return pathResolver.relativePath(fileSystem.baseDir(), file);
   }
 
-  @CheckForNull
-  private InputFile newInputFile(ModuleFileSystem fileSystem, String type, File file, String path) {
-
-    Map<String, String> attributes = Maps.newHashMap();
-    set(attributes, InputFile.ATTRIBUTE_TYPE, type);
-
-    String resourceKey = PathUtils.sanitize(path);
-    set(attributes, DefaultInputFile.ATTRIBUTE_COMPONENT_KEY, module.getEffectiveKey() + ":" + resourceKey);
-    // hash + status
-    initStatus(file, fileSystem.sourceCharset(), path, attributes);
-
-    DefaultInputFile inputFile = DefaultInputFile.create(file, fileSystem.sourceCharset(), path, attributes);
-    String lang = languageRecognizer.of(inputFile);
-    if (lang == null) {
-      return null;
-    }
-    set(inputFile.attributes(), InputFile.ATTRIBUTE_LANGUAGE, lang);
-
-    setDeprecatedAttributes(fileSystem, type, file, attributes, inputFile, lang);
-
-    return inputFile;
-  }
-
-  private void setDeprecatedAttributes(ModuleFileSystem fileSystem, String type, File file, Map<String, String> attributes, DefaultInputFile inputFile, String lang) {
-    List<File> sourceDirs = InputFile.TYPE_MAIN.equals(type) ? fileSystem.sourceDirs() : fileSystem.testDirs();
-    for (File src : sourceDirs) {
-      String sourceRelativePath = pathResolver.relativePath(src, file);
-      if (sourceRelativePath != null) {
-        set(attributes, DefaultInputFile.ATTRIBUTE_SOURCEDIR_PATH, PathUtils.canonicalPath(src));
-        set(attributes, DefaultInputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, sourceRelativePath);
-        if (Java.KEY.equals(lang)) {
-          set(inputFile.attributes(), DefaultInputFile.ATTRIBUTE_COMPONENT_DEPRECATED_KEY, module.getEffectiveKey() + ":"
-            + JavaFile.fromRelativePath(sourceRelativePath, false).getDeprecatedKey());
-        } else {
-          set(inputFile.attributes(), DefaultInputFile.ATTRIBUTE_COMPONENT_DEPRECATED_KEY, module.getEffectiveKey() + ":" + sourceRelativePath);
-        }
-        return;
-      }
-    }
-  }
-
-  private void initStatus(File file, Charset charset, String baseRelativePath, Map<String, String> attributes) {
-    String hash = fileHashes.hash(file, charset);
-    set(attributes, DefaultInputFile.ATTRIBUTE_HASH, hash);
-
-    String remoteHash = fileHashes.remoteHash(baseRelativePath);
-    // currently no need to store this remote hash in attributes
-    if (StringUtils.equals(hash, remoteHash)) {
-      set(attributes, InputFile.ATTRIBUTE_STATUS, InputFile.STATUS_SAME);
-    } else if (StringUtils.isEmpty(remoteHash)) {
-      set(attributes, InputFile.ATTRIBUTE_STATUS, InputFile.STATUS_ADDED);
-    } else {
-      set(attributes, InputFile.ATTRIBUTE_STATUS, InputFile.STATUS_CHANGED);
-    }
-  }
-
-  private void set(Map<String, String> attributes, String key, @Nullable String value) {
-    if (value != null) {
-      attributes.put(key, value);
-    }
-  }
 
   private boolean accept(InputFile inputFile) {
     // InputFileFilter extensions

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileHashDigest.java → sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java

@@ -31,47 +31,60 @@ import java.security.MessageDigest;
  * Computes hash of files. Ends of Lines are ignored, so files with
  * same content but different EOL encoding have the same hash.
  */
-class FileHashDigest {
+class FileMetadata {
+
+  private static final char LINE_FEED = '\n';
+  private static final char CARRIAGE_RETURN = '\r';
 
   // This singleton aims only to increase the coverage by allowing
   // to test the private method !
-  static final FileHashDigest INSTANCE = new FileHashDigest();
+  static final FileMetadata INSTANCE = new FileMetadata();
 
-  private FileHashDigest() {
+  private FileMetadata() {
   }
 
   /**
    * Compute hash of a file ignoring line ends differences.
    * Maximum performance is needed.
    */
-  String hash(File file, Charset charset) {
+  Metadata read(File file, Charset encoding) {
     Reader reader = null;
+    long lines = 0L;
+    char c = (char)-1;
     try {
       MessageDigest md5Digest = DigestUtils.getMd5Digest();
       md5Digest.reset();
-      reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
+      reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), encoding));
       int i = reader.read();
       boolean afterCR = true;
       while (i != -1) {
-        char c = (char) i;
+        c = (char) i;
         if (afterCR) {
           afterCR = false;
-          if (c == '\n') {
+          if (c == LINE_FEED) {
             // Ignore
             i = reader.read();
+            lines++;
             continue;
           }
         }
-        if (c == '\r') {
+        if (c == CARRIAGE_RETURN) {
           afterCR = true;
-          c = '\n';
+          c = LINE_FEED;
+        } else if (c == LINE_FEED) {
+          lines++;
         }
         md5Digest.update(charToBytesUTF(c));
         i = reader.read();
       }
-      return Hex.encodeHexString(md5Digest.digest());
+      if (c != LINE_FEED) {
+        lines++;
+      }
+      String hash = Hex.encodeHexString(md5Digest.digest());
+      return new Metadata(lines, hash);
+
     } catch (IOException e) {
-      throw new IllegalStateException(String.format("Fail to compute hash of file %s with charset %s", file.getAbsolutePath(), charset), e);
+      throw new IllegalStateException(String.format("Fail to read file '%s' with encoding '%s'", file.getAbsolutePath(), encoding), e);
     } finally {
       IOUtils.closeQuietly(reader);
     }
@@ -87,4 +100,14 @@ class FileHashDigest {
     }
     return b;
   }
+
+  static class Metadata {
+    long lines;
+    String hash;
+
+    private Metadata(long lines, String hash) {
+      this.lines = lines;
+      this.hash = hash;
+    }
+  }
 }

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java

@@ -0,0 +1,91 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import com.google.common.collect.Maps;
+import org.apache.commons.io.FilenameUtils;
+import org.sonar.api.resources.Java;
+import org.sonar.api.resources.JavaFile;
+import org.sonar.api.scan.filesystem.InputFile;
+import org.sonar.api.scan.filesystem.PathResolver;
+import org.sonar.api.scan.filesystem.internal.DefaultInputFile;
+import org.sonar.api.utils.MessageException;
+
+import javax.annotation.CheckForNull;
+import java.io.File;
+import java.util.List;
+
+class InputFileBuilder {
+
+  private final String moduleKey;
+  private final PathResolver pathResolver;
+  private final LanguageDetection langDetection;
+  private final StatusDetection statusDetection;
+  private final DefaultModuleFileSystem fs;
+
+  InputFileBuilder(String moduleKey, PathResolver pathResolver, LanguageDetection langDetection,
+                   StatusDetection statusDetection, DefaultModuleFileSystem fs) {
+    this.moduleKey = moduleKey;
+    this.pathResolver = pathResolver;
+    this.langDetection = langDetection;
+    this.statusDetection = statusDetection;
+    this.fs = fs;
+  }
+
+  @CheckForNull
+  DefaultInputFile create(File file, String type) {
+    String relativePath = pathResolver.relativePath(fs.baseDir(), file);
+    if (relativePath == null) {
+      throw MessageException.of(String.format("File '%s' is ignored. It is not in module basedir '%s'.", file.getAbsolutePath(), fs.baseDir()));
+    }
+    DefaultInputFile inputFile = DefaultInputFile.create(file, fs.sourceCharset(), relativePath, Maps.<String, String>newHashMap());
+    inputFile.setType(type);
+    inputFile.setKey(moduleKey + ":" + inputFile.path());
+    String lang = langDetection.language(inputFile);
+    if (lang == null) {
+      // TODO use a default plain-text language ?
+      return null;
+    }
+    inputFile.setLanguage(lang);
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(inputFile.file(), fs.sourceCharset());
+    inputFile.setLines(metadata.lines);
+    inputFile.setHash(metadata.hash);
+    inputFile.setStatus(statusDetection.status(inputFile.path(), metadata.hash));
+    fillDeprecatedData(inputFile);
+    return inputFile;
+  }
+
+  private void fillDeprecatedData(DefaultInputFile inputFile) {
+    List<File> sourceDirs = InputFile.TYPE_MAIN.equals(inputFile.type()) ? fs.sourceDirs() : fs.testDirs();
+    for (File sourceDir : sourceDirs) {
+      String sourceRelativePath = pathResolver.relativePath(sourceDir, inputFile.file());
+      if (sourceRelativePath != null) {
+        inputFile.setPathRelativeToSourceDir(sourceRelativePath);
+        inputFile.setSourceDirAbsolutePath(FilenameUtils.normalize(sourceDir.getAbsolutePath(), true));
+
+        if (Java.KEY.equals(inputFile.language())) {
+          inputFile.setDeprecatedKey(moduleKey + ":" + JavaFile.fromRelativePath(sourceRelativePath, false).getDeprecatedKey());
+        } else {
+          inputFile.setDeprecatedKey(moduleKey + ":" + sourceRelativePath);
+        }
+      }
+    }
+  }
+}

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactory.java

@@ -0,0 +1,44 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import org.sonar.api.BatchComponent;
+import org.sonar.api.resources.Project;
+import org.sonar.api.scan.filesystem.PathResolver;
+
+public class InputFileBuilderFactory implements BatchComponent {
+
+  private final String moduleKey;
+  private final PathResolver pathResolver;
+  private final LanguageDetectionFactory langDetectionFactory;
+  private final StatusDetectionFactory statusDetectionFactory;
+
+  public InputFileBuilderFactory(Project moduleDef, PathResolver pathResolver, LanguageDetectionFactory langDetectionFactory,
+                                 StatusDetectionFactory statusDetectionFactory) {
+    this.moduleKey = moduleDef.getEffectiveKey();
+    this.pathResolver = pathResolver;
+    this.langDetectionFactory = langDetectionFactory;
+    this.statusDetectionFactory = statusDetectionFactory;
+  }
+
+  InputFileBuilder create(DefaultModuleFileSystem fs) {
+    return new InputFileBuilder(moduleKey, pathResolver, langDetectionFactory.create(), statusDetectionFactory.create(), fs);
+  }
+}

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/LanguageRecognizer.java → sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/LanguageDetection.java

@@ -20,55 +20,45 @@
 package org.sonar.batch.scan.filesystem;
 
 import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.lang.StringUtils;
-import org.picocontainer.Startable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.sonar.api.BatchComponent;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Language;
 import org.sonar.api.resources.Languages;
 import org.sonar.api.scan.filesystem.InputFile;
-import org.sonar.api.utils.SonarException;
+import org.sonar.api.utils.MessageException;
 
 import javax.annotation.CheckForNull;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
 /**
- * Detect language of source files.
+ * Detect language of a source file based on its suffix and configured patterns.
  */
-public class LanguageRecognizer implements BatchComponent, Startable {
-
-  private static final Logger LOG = LoggerFactory.getLogger(LanguageRecognizer.class);
+class LanguageDetection {
 
-  private final Languages languages;
+  private static final Logger LOG = LoggerFactory.getLogger(LanguageDetection.class);
 
   /**
    * Lower-case extension -> languages
    */
-  private Map<String, PathPattern[]> patternByLanguage = Maps.newLinkedHashMap();
-
-  private Settings settings;
+  private final Map<String, PathPattern[]> patternByLanguage = Maps.newLinkedHashMap();
+  private final List<String> languagesToConsider = Lists.newArrayList();
+  private final String forcedLanguage;
 
-  public LanguageRecognizer(Settings settings, Languages languages) {
-    this.settings = settings;
-    this.languages = languages;
-  }
-
-  @Override
-  public void start() {
+  LanguageDetection(Settings settings, Languages languages) {
     for (Language language : languages.all()) {
       String[] filePatterns = settings.getStringArray(getFileLangPatternPropKey(language.getKey()));
       PathPattern[] pathPatterns = PathPattern.create(filePatterns);
       if (pathPatterns.length > 0) {
         patternByLanguage.put(language.getKey(), pathPatterns);
-      } else if (language.getFileSuffixes().length > 0) {
+      } else {
         // If no custom language pattern is defined then fallback to suffixes declared by language
         String[] patterns = Arrays.copyOf(language.getFileSuffixes(), language.getFileSuffixes().length);
         for (int i = 0; i < patterns.length; i++) {
@@ -81,28 +71,21 @@ public class LanguageRecognizer implements BatchComponent, Startable {
         LOG.debug("Declared extensions of language " + language + " were converted to " + getDetails(language.getKey()));
       }
     }
-  }
-
-  private String getFileLangPatternPropKey(String languageKey) {
-    return "sonar.lang.patterns." + languageKey;
-  }
-
-  @Override
-  public void stop() {
-    // do nothing
-  }
-
-  @CheckForNull
-  String of(InputFile inputFile) {
-    String deprecatedLanguageParam = settings.getString(CoreProperties.PROJECT_LANGUAGE_PROPERTY);
 
+    forcedLanguage = StringUtils.defaultIfBlank(settings.getString(CoreProperties.PROJECT_LANGUAGE_PROPERTY), null);
     // First try with lang patterns
-    List<String> languagesToConsider = new ArrayList<String>();
-    if (!StringUtils.isBlank(deprecatedLanguageParam)) {
-      languagesToConsider.add(deprecatedLanguageParam);
+    if (forcedLanguage != null) {
+      if (!patternByLanguage.containsKey(forcedLanguage)) {
+        throw MessageException.of("No language is installed with key '" + forcedLanguage + "'. Please update property '" + CoreProperties.PROJECT_LANGUAGE_PROPERTY + "'");
+      }
+      languagesToConsider.add(forcedLanguage);
     } else {
       languagesToConsider.addAll(patternByLanguage.keySet());
     }
+  }
+
+  @CheckForNull
+  String language(InputFile inputFile) {
     String detectedLanguage = null;
     for (String languageKey : languagesToConsider) {
       PathPattern[] patterns = patternByLanguage.get(languageKey);
@@ -114,7 +97,7 @@ public class LanguageRecognizer implements BatchComponent, Startable {
               break;
             } else {
               // Language was already forced by another pattern
-              throw new SonarException("Language of file '" + inputFile.path() + "' can not be decided as the file matches patterns of both " + getDetails(detectedLanguage)
+              throw MessageException.of("Language of file '" + inputFile.path() + "' can not be decided as the file matches patterns of both " + getDetails(detectedLanguage)
                 + " and " + getDetails(languageKey));
             }
           }
@@ -127,22 +110,19 @@ public class LanguageRecognizer implements BatchComponent, Startable {
     }
 
     // Check if deprecated sonar.language is used and we are on a language without declared extensions
-    if (StringUtils.isNotBlank(deprecatedLanguageParam)) {
-      Language language = languages.get(deprecatedLanguageParam);
-      if (language == null) {
-        throw new SonarException("No language is installed with key '" + deprecatedLanguageParam + "'. Please update property '" + CoreProperties.PROJECT_LANGUAGE_PROPERTY + "'");
-      }
+    if (forcedLanguage != null) {
       // Languages without declared suffixes match everything
-      String[] fileSuffixes = language.getFileSuffixes();
-      if (fileSuffixes.length == 0) {
-        return deprecatedLanguageParam;
+      if (patternByLanguage.get(forcedLanguage).length == 0) {
+        return forcedLanguage;
       }
-      return null;
     }
-
     return null;
   }
 
+  private String getFileLangPatternPropKey(String languageKey) {
+    return "sonar.lang.patterns." + languageKey;
+  }
+
   private String getDetails(String detectedLanguage) {
     return getFileLangPatternPropKey(detectedLanguage) + " : " + Joiner.on(",").join(patternByLanguage.get(detectedLanguage));
   }

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/LanguageDetectionFactory.java

@@ -0,0 +1,38 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import org.sonar.api.BatchComponent;
+import org.sonar.api.config.Settings;
+import org.sonar.api.resources.Languages;
+
+public class LanguageDetectionFactory implements BatchComponent {
+  private final Settings settings;
+  private final Languages languages;
+
+  public LanguageDetectionFactory(Settings settings, Languages languages) {
+    this.settings = settings;
+    this.languages = languages;
+  }
+
+  public LanguageDetection create() {
+    return new LanguageDetection(settings, languages);
+  }
+}

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/RemoteFileHashes.java → sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/PreviousFileHashLoader.java

@@ -36,23 +36,23 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Map;
 
-public class RemoteFileHashes implements BatchComponent, Startable {
+public class PreviousFileHashLoader implements BatchComponent {
 
   private final SnapshotDataDao dao;
   private final PastSnapshotFinder pastSnapshotFinder;
   private final Snapshot snapshot;
 
-  private Map<String, String> pathToHash = Maps.newHashMap();
-
-  public RemoteFileHashes(Snapshot snapshot, SnapshotDataDao dao, PastSnapshotFinder pastSnapshotFinder) {
+  public PreviousFileHashLoader(Snapshot snapshot, SnapshotDataDao dao, PastSnapshotFinder pastSnapshotFinder) {
     this.snapshot = snapshot;
     this.dao = dao;
     this.pastSnapshotFinder = pastSnapshotFinder;
   }
 
-  @Override
-  public void start() {
-    // Extract previous checksum of all files of this module and store them in a map
+  /**
+   * Extract hash of the files parsed during the previous analysis
+   */
+  Map<String, String> hashByRelativePath() {
+    Map<String, String> map = Maps.newHashMap();
     PastSnapshot pastSnapshot = pastSnapshotFinder.findPreviousAnalysis(snapshot);
     if (pastSnapshot.isRelatedToSnapshot()) {
       Collection<SnapshotDataDto> selectSnapshotData = dao.selectSnapshotData(
@@ -62,18 +62,10 @@ public class RemoteFileHashes implements BatchComponent, Startable {
       if (!selectSnapshotData.isEmpty()) {
         SnapshotDataDto snapshotDataDto = selectSnapshotData.iterator().next();
         String data = snapshotDataDto.getData();
-        pathToHash = KeyValueFormat.parse(data);
+        map = KeyValueFormat.parse(data);
       }
     }
+    return map;
   }
 
-  @CheckForNull
-  public String remoteHash(String baseRelativePath) {
-    return pathToHash.get(baseRelativePath);
-  }
-
-  @Override
-  public void stop() {
-    // nothing to do
-  }
 }

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetection.java

@@ -0,0 +1,46 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import org.apache.commons.lang.StringUtils;
+import org.sonar.api.scan.filesystem.InputFile;
+
+import java.util.Map;
+
+class StatusDetection {
+
+  private final Map<String, String> previousHashByRelativePath;
+
+  StatusDetection(Map<String, String> previousHashByRelativePath) {
+    this.previousHashByRelativePath = previousHashByRelativePath;
+  }
+
+  String status(String relativePath, String hash) {
+    String previousHash = previousHashByRelativePath.get(relativePath);
+
+    if (StringUtils.equals(hash, previousHash)) {
+      return InputFile.STATUS_SAME;
+    }
+    if (StringUtils.isEmpty(previousHash)) {
+      return InputFile.STATUS_ADDED;
+    }
+    return InputFile.STATUS_CHANGED;
+  }
+}

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileHashes.java → sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetectionFactory.java

@@ -21,28 +21,15 @@ package org.sonar.batch.scan.filesystem;
 
 import org.sonar.api.BatchComponent;
 
-import javax.annotation.CheckForNull;
-import java.io.File;
-import java.nio.charset.Charset;
+public class StatusDetectionFactory implements BatchComponent {
 
-/**
- * Facade for local and remote file hashes
- */
-public class FileHashes implements BatchComponent {
-
-  private final RemoteFileHashes remoteFileHashes;
-
-  public FileHashes(RemoteFileHashes remoteFileHashes) {
-    this.remoteFileHashes = remoteFileHashes;
-  }
+  private final PreviousFileHashLoader previousFileHashLoader;
 
-  @CheckForNull
-  public String hash(File file, Charset charset) {
-    return FileHashDigest.INSTANCE.hash(file, charset);
+  public StatusDetectionFactory(PreviousFileHashLoader l) {
+    this.previousFileHashLoader = l;
   }
 
-  @CheckForNull
-  public String remoteHash(String baseRelativePath) {
-    return remoteFileHashes.remoteHash(baseRelativePath);
+  StatusDetection create() {
+    return new StatusDetection(previousFileHashLoader.hashByRelativePath());
   }
 }

sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileHashDigestTest.java

@@ -1,82 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2013 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.scan.filesystem;
-
-import com.google.common.base.Charsets;
-import org.apache.commons.io.FileUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-
-import static org.fest.assertions.Assertions.assertThat;
-
-public class FileHashDigestTest {
-
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
-
-  @Rule
-  public TemporaryFolder temp = new TemporaryFolder();
-
-  @Test
-  public void should_compute_hash() throws Exception {
-    File tempFile = temp.newFile();
-    FileUtils.write(tempFile, "foo\r\nbar", Charsets.UTF_8, true);
-
-    assertThat(FileHashDigest.INSTANCE.hash(tempFile, Charsets.UTF_8)).isEqualTo("daef8a22a3f12580beadf086a9e11519");
-  }
-
-  @Test
-  public void should_normalize_line_ends() throws Exception {
-    File file1 = temp.newFile();
-    FileUtils.write(file1, "foobar\nfofo", Charsets.UTF_8);
-    String hash1 = FileHashDigest.INSTANCE.hash(file1, Charsets.UTF_8);
-
-    File file2 = temp.newFile();
-    FileUtils.write(file2, "foobar\r\nfofo", Charsets.UTF_8);
-    String hash2 = FileHashDigest.INSTANCE.hash(file2, Charsets.UTF_8);
-
-    File file3 = temp.newFile();
-    FileUtils.write(file3, "foobar\rfofo", Charsets.UTF_8);
-    String hash3 = FileHashDigest.INSTANCE.hash(file3, Charsets.UTF_8);
-
-    File file4 = temp.newFile();
-    FileUtils.write(file4, "foobar\nfofo\n", Charsets.UTF_8);
-    String hash4 = FileHashDigest.INSTANCE.hash(file4, Charsets.UTF_8);
-
-    assertThat(hash1).isEqualTo(hash2);
-    assertThat(hash1).isEqualTo(hash3);
-    assertThat(hash1).isNotEqualTo(hash4);
-  }
-
-  @Test
-  public void should_throw_if_file_does_not_exist() throws Exception {
-    File tempFolder = temp.newFolder();
-    File file = new File(tempFolder, "doesNotExist.txt");
-
-    thrown.expect(IllegalStateException.class);
-    thrown.expectMessage("Fail to compute hash of file " + file.getAbsolutePath() + " with charset UTF-8");
-
-    FileHashDigest.INSTANCE.hash(file, Charsets.UTF_8);
-  }
-}

sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileHashesTest.java

@@ -1,58 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2013 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.scan.filesystem;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.nio.charset.Charset;
-
-import static org.fest.assertions.Assertions.assertThat;
-import static org.mockito.Mockito.*;
-
-public class FileHashesTest {
-
-  @Rule
-  public TemporaryFolder temp = new TemporaryFolder();
-
-  RemoteFileHashes remoteFileHashes = mock(RemoteFileHashes.class);
-
-  @Test
-  public void hash() throws Exception {
-    File file = temp.newFile();
-    FileUtils.write(file, "fooo");
-
-    FileHashes hashes = new FileHashes(remoteFileHashes);
-    assertThat(hashes.hash(file, Charset.forName("UTF-8"))).isEqualTo("efc4470c96a94b1ff400175ef8368444");
-    verifyZeroInteractions(remoteFileHashes);
-  }
-
-  @Test
-  public void remote_hash() throws Exception {
-    String path = "src/main/java/Foo.java";
-    when(remoteFileHashes.remoteHash(path)).thenReturn("ABCDE");
-
-    FileHashes hashes = new FileHashes(remoteFileHashes);
-    assertThat(hashes.remoteHash(path)).isEqualTo("ABCDE");
-  }
-}

sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileIndexTest.java

@@ -1,71 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2013 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.scan.filesystem;
-
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.sonar.api.resources.Project;
-import org.sonar.api.scan.filesystem.InputDir;
-import org.sonar.api.scan.filesystem.InputFile;
-import org.sonar.api.scan.filesystem.PathResolver;
-import org.sonar.api.scan.filesystem.internal.DefaultInputDir;
-
-import java.io.File;
-
-import static org.fest.assertions.Assertions.assertThat;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class FileIndexTest {
-
-  @Rule
-  public TemporaryFolder temp = new TemporaryFolder();
-
-  @Test
-  public void should_return_inputDir() throws Exception {
-    FileIndex index = new FileIndex(null, null, null, null, null, new PathResolver(), new Project("myProject"));
-    File baseDir = temp.newFolder();
-    DefaultModuleFileSystem fileSystem = mock(DefaultModuleFileSystem.class);
-    when(fileSystem.baseDir()).thenReturn(baseDir);
-    File ioFile = new File(baseDir, "src/main/java/com/foo");
-    InputDir inputDir = index.inputDir(fileSystem, ioFile);
-
-    assertThat(inputDir.name()).isEqualTo("src/main/java/com/foo");
-    assertThat(inputDir.file()).isEqualTo(ioFile);
-    assertThat(inputDir.attribute(DefaultInputDir.ATTRIBUTE_COMPONENT_KEY)).isEqualTo("myProject:src/main/java/com/foo");
-  }
-
-  @Test
-  public void should_not_index_aggregator() throws Exception {
-    Project project = new Project("myProject");
-    new Project("moduleA").setParent(project);
-    InputFileCache fileCache = mock(InputFileCache.class);
-    FileIndex index = new FileIndex(null, null, null, fileCache, null, new PathResolver(), project);
-
-    index.index(null);
-
-    verify(fileCache, never()).put(anyString(), any(InputFile.class));
-  }
-}

sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java

@@ -0,0 +1,114 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.FileUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class FileMetadataTest {
+
+  private static final String EXPECTED_HASH_WITHOUT_LATEST_EOL = "c80cc50d65ace6c4eb63f189d274dbeb";
+  private static final String EXPECTED_HASH_WITH_LATEST_EOL = "bf77e51d219e7d7d643faac86f1b5d15";
+
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  @Test
+  public void windows_without_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\r\nbar\r\nbaz", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3L);
+    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+  }
+
+  @Test
+  public void windows_with_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\r\nbar\r\nbaz\r\n", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3L);
+    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
+  }
+
+  @Test
+  public void unix_without_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\nbar\nbaz", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3L);
+    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+  }
+
+  @Test
+  public void unix_with_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\nbar\nbaz\n", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3L);
+    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
+  }
+
+  @Test
+  public void mix_of_newlines_with_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\nbar\r\nbaz\n", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3L);
+    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
+  }
+
+  @Test
+  public void mix_of_newlines_without_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\nbar\r\nbaz", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3L);
+    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+  }
+
+  @Test
+  public void should_throw_if_file_does_not_exist() throws Exception {
+    File tempFolder = temp.newFolder();
+    File file = new File(tempFolder, "doesNotExist.txt");
+
+    thrown.expect(IllegalStateException.class);
+    thrown.expectMessage("Fail to read file '" + file.getAbsolutePath() + "' with encoding 'UTF-8'");
+
+    FileMetadata.INSTANCE.read(file, Charsets.UTF_8);
+  }
+}

sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/LanguageDetectionTest.java

@@ -0,0 +1,212 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import com.google.common.base.Charsets;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.config.Settings;
+import org.sonar.api.resources.Language;
+import org.sonar.api.resources.Languages;
+import org.sonar.api.scan.filesystem.InputFile;
+import org.sonar.api.scan.filesystem.internal.InputFileBuilder;
+import org.sonar.api.utils.MessageException;
+
+import java.io.File;
+import java.io.IOException;
+
+import static junit.framework.Assert.fail;
+import static org.fest.assertions.Assertions.assertThat;
+import static org.mockito.Mockito.spy;
+
+public class LanguageDetectionTest {
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  @Test
+  public void test_sanitizeExtension() throws Exception {
+    assertThat(LanguageDetection.sanitizeExtension(".cbl")).isEqualTo("cbl");
+    assertThat(LanguageDetection.sanitizeExtension(".CBL")).isEqualTo("cbl");
+    assertThat(LanguageDetection.sanitizeExtension("CBL")).isEqualTo("cbl");
+    assertThat(LanguageDetection.sanitizeExtension("cbl")).isEqualTo("cbl");
+  }
+
+  @Test
+  public void search_by_file_extension() throws Exception {
+    Languages languages = new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob"));
+    LanguageDetection detection = new LanguageDetection(new Settings(), languages);
+
+    assertThat(detection.language(newInputFile("Foo.java"))).isEqualTo("java");
+    assertThat(detection.language(newInputFile("src/Foo.java"))).isEqualTo("java");
+    assertThat(detection.language(newInputFile("Foo.JAVA"))).isEqualTo("java");
+    assertThat(detection.language(newInputFile("Foo.jav"))).isEqualTo("java");
+    assertThat(detection.language(newInputFile("Foo.Jav"))).isEqualTo("java");
+
+    assertThat(detection.language(newInputFile("abc.cbl"))).isEqualTo("cobol");
+    assertThat(detection.language(newInputFile("abc.CBL"))).isEqualTo("cobol");
+
+    assertThat(detection.language(newInputFile("abc.php"))).isNull();
+    assertThat(detection.language(newInputFile("abc"))).isNull();
+  }
+
+  @Test
+  public void should_not_fail_if_no_language() throws Exception {
+    LanguageDetection detection = spy(new LanguageDetection(new Settings(), new Languages()));
+    assertThat(detection.language(newInputFile("Foo.java"))).isNull();
+  }
+
+  @Test
+  public void plugin_can_declare_a_file_extension_twice_for_case_sensitivity() throws Exception {
+    Languages languages = new Languages(new MockLanguage("abap", "abap", "ABAP"));
+
+    LanguageDetection detection = new LanguageDetection(new Settings(), languages);
+    assertThat(detection.language(newInputFile("abc.abap"))).isEqualTo("abap");
+  }
+
+  @Test
+  public void language_with_no_extension() throws Exception {
+    // abap does not declare any file extensions.
+    // When analyzing an ABAP project, then all source files must be parsed.
+    Languages languages = new Languages(new MockLanguage("java", "java"), new MockLanguage("abap"));
+
+    // No side-effect on non-ABAP projects
+    LanguageDetection detection = new LanguageDetection(new Settings(), languages);
+    assertThat(detection.language(newInputFile("abc"))).isNull();
+    assertThat(detection.language(newInputFile("abc.abap"))).isNull();
+    assertThat(detection.language(newInputFile("abc.java"))).isEqualTo("java");
+
+    Settings settings = new Settings();
+    settings.setProperty(CoreProperties.PROJECT_LANGUAGE_PROPERTY, "abap");
+    detection = new LanguageDetection(settings, languages);
+    assertThat(detection.language(newInputFile("abc"))).isEqualTo("abap");
+    assertThat(detection.language(newInputFile("abc.txt"))).isEqualTo("abap");
+    assertThat(detection.language(newInputFile("abc.java"))).isEqualTo("abap");
+  }
+
+  @Test
+  public void force_language_using_deprecated_property() throws Exception {
+    Languages languages = new Languages(new MockLanguage("java", "java"), new MockLanguage("php", "php"));
+
+    Settings settings = new Settings();
+    settings.setProperty(CoreProperties.PROJECT_LANGUAGE_PROPERTY, "java");
+    LanguageDetection detection = new LanguageDetection(settings, languages);
+    assertThat(detection.language(newInputFile("abc"))).isNull();
+    assertThat(detection.language(newInputFile("abc.php"))).isNull();
+    assertThat(detection.language(newInputFile("abc.java"))).isEqualTo("java");
+    assertThat(detection.language(newInputFile("src/abc.java"))).isEqualTo("java");
+  }
+
+  @Test
+  public void fail_if_invalid_language() throws Exception {
+    thrown.expect(MessageException.class);
+    thrown.expectMessage("No language is installed with key 'unknown'. Please update property 'sonar.language'");
+
+    Languages languages = new Languages(new MockLanguage("java", "java"), new MockLanguage("php", "php"));
+    Settings settings = new Settings();
+    settings.setProperty(CoreProperties.PROJECT_LANGUAGE_PROPERTY, "unknown");
+    new LanguageDetection(settings, languages);
+  }
+
+  @Test
+  public void fail_if_conflicting_language_suffix() throws Exception {
+    Languages languages = new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml"));
+    LanguageDetection detection = new LanguageDetection(new Settings(), languages);
+    try {
+      detection.language(newInputFile("abc.xhtml"));
+      fail();
+    } catch (MessageException e) {
+      assertThat(e.getMessage())
+        .contains("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ")
+        .contains("sonar.lang.patterns.web : **/*.xhtml")
+        .contains("sonar.lang.patterns.xml : **/*.xhtml");
+    }
+  }
+
+  @Test
+  public void solve_conflict_using_filepattern() throws Exception {
+    Languages languages = new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml"));
+
+    Settings settings = new Settings();
+    settings.setProperty("sonar.lang.patterns.xml", "xml/**");
+    settings.setProperty("sonar.lang.patterns.web", "web/**");
+    LanguageDetection detection = new LanguageDetection(settings, languages);
+    assertThat(detection.language(newInputFile("xml/abc.xhtml"))).isEqualTo("xml");
+    assertThat(detection.language(newInputFile("web/abc.xhtml"))).isEqualTo("web");
+  }
+
+  @Test
+  public void fail_if_conflicting_filepattern() throws Exception {
+    Languages languages = new Languages(new MockLanguage("abap", "abap"), new MockLanguage("cobol", "cobol"));
+    Settings settings = new Settings();
+    settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt");
+    settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt");
+
+    LanguageDetection detection = new LanguageDetection(settings, languages);
+
+    assertThat(detection.language(newInputFile("abc.abap"))).isEqualTo("abap");
+    assertThat(detection.language(newInputFile("abc.cobol"))).isEqualTo("cobol");
+    try {
+      detection.language(newInputFile("abc.txt"));
+      fail();
+    } catch (MessageException e) {
+      assertThat(e.getMessage())
+        .contains("Language of file 'abc.txt' can not be decided as the file matches patterns of both ")
+        .contains("sonar.lang.patterns.abap : *.abap,*.txt")
+        .contains("sonar.lang.patterns.cobol : *.cobol,*.txt");
+    }
+  }
+
+  private InputFile newInputFile(String path) throws IOException {
+    File basedir = temp.newFolder();
+    return new InputFileBuilder(new File(basedir, path), Charsets.UTF_8, path).build();
+  }
+
+  static class MockLanguage implements Language {
+    private final String key;
+    private final String[] extensions;
+
+    MockLanguage(String key, String... extensions) {
+      this.key = key;
+      this.extensions = extensions;
+    }
+
+    @Override
+    public String getKey() {
+      return key;
+    }
+
+    @Override
+    public String getName() {
+      return key;
+    }
+
+    @Override
+    public String[] getFileSuffixes() {
+      return extensions;
+    }
+  }
+}

sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/LanguageRecognizerTest.java

@@ -1,245 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2013 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.scan.filesystem;
-
-import com.google.common.base.Charsets;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.config.Settings;
-import org.sonar.api.resources.Language;
-import org.sonar.api.resources.Languages;
-import org.sonar.api.scan.filesystem.InputFile;
-import org.sonar.api.scan.filesystem.internal.InputFileBuilder;
-import org.sonar.api.utils.SonarException;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.fest.assertions.Assertions.assertThat;
-import static org.mockito.Mockito.spy;
-
-public class LanguageRecognizerTest {
-
-  @Rule
-  public TemporaryFolder temp = new TemporaryFolder();
-
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
-
-  @Test
-  public void test_sanitizeExtension() throws Exception {
-    assertThat(LanguageRecognizer.sanitizeExtension(".cbl")).isEqualTo("cbl");
-    assertThat(LanguageRecognizer.sanitizeExtension(".CBL")).isEqualTo("cbl");
-    assertThat(LanguageRecognizer.sanitizeExtension("CBL")).isEqualTo("cbl");
-    assertThat(LanguageRecognizer.sanitizeExtension("cbl")).isEqualTo("cbl");
-  }
-
-  @Test
-  public void search_by_file_extension() throws Exception {
-    Languages languages = new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob"));
-    LanguageRecognizer recognizer = new LanguageRecognizer(new Settings(), languages);
-
-    recognizer.start();
-    assertThat(recognizer.of(newInputFile("Foo.java"))).isEqualTo("java");
-    assertThat(recognizer.of(newInputFile("src/Foo.java"))).isEqualTo("java");
-    assertThat(recognizer.of(newInputFile("Foo.JAVA"))).isEqualTo("java");
-    assertThat(recognizer.of(newInputFile("Foo.jav"))).isEqualTo("java");
-    assertThat(recognizer.of(newInputFile("Foo.Jav"))).isEqualTo("java");
-
-    assertThat(recognizer.of(newInputFile("abc.cbl"))).isEqualTo("cobol");
-    assertThat(recognizer.of(newInputFile("abc.CBL"))).isEqualTo("cobol");
-
-    assertThat(recognizer.of(newInputFile("abc.php"))).isNull();
-    assertThat(recognizer.of(newInputFile("abc"))).isNull();
-    recognizer.stop();
-  }
-
-  @Test
-  public void should_not_fail_if_no_language() throws Exception {
-    LanguageRecognizer recognizer = spy(new LanguageRecognizer(new Settings(), new Languages()));
-    recognizer.start();
-    assertThat(recognizer.of(newInputFile("Foo.java"))).isNull();
-  }
-
-  @Test
-  public void plugin_can_declare_a_file_extension_twice_for_case_sensitivity() throws Exception {
-    Languages languages = new Languages(new MockLanguage("abap", "abap", "ABAP"));
-
-    LanguageRecognizer recognizer = new LanguageRecognizer(new Settings(), languages);
-    recognizer.start();
-    assertThat(recognizer.of(newInputFile("abc.abap"))).isEqualTo("abap");
-  }
-
-  @Test
-  public void language_with_no_extension() throws Exception {
-    // abap does not declare any file extensions.
-    // When analyzing an ABAP project, then all source files must be parsed.
-    Languages languages = new Languages(new MockLanguage("java", "java"), new MockLanguage("abap"));
-
-    // No side-effect on non-ABAP projects
-    LanguageRecognizer recognizer = new LanguageRecognizer(new Settings(), languages);
-    recognizer.start();
-    assertThat(recognizer.of(newInputFile("abc"))).isNull();
-    assertThat(recognizer.of(newInputFile("abc.abap"))).isNull();
-    assertThat(recognizer.of(newInputFile("abc.java"))).isEqualTo("java");
-    recognizer.stop();
-
-    Settings settings = new Settings();
-    settings.setProperty(CoreProperties.PROJECT_LANGUAGE_PROPERTY, "abap");
-    recognizer = new LanguageRecognizer(settings, languages);
-    recognizer.start();
-    assertThat(recognizer.of(newInputFile("abc"))).isEqualTo("abap");
-    assertThat(recognizer.of(newInputFile("abc.txt"))).isEqualTo("abap");
-    assertThat(recognizer.of(newInputFile("abc.java"))).isEqualTo("abap");
-    recognizer.stop();
-  }
-
-  @Test
-  public void force_language_using_deprecated_property() throws Exception {
-    Languages languages = new Languages(new MockLanguage("java", "java"), new MockLanguage("php", "php"));
-
-    Settings settings = new Settings();
-    settings.setProperty(CoreProperties.PROJECT_LANGUAGE_PROPERTY, "java");
-    LanguageRecognizer recognizer = new LanguageRecognizer(settings, languages);
-    recognizer.start();
-    assertThat(recognizer.of(newInputFile("abc"))).isNull();
-    assertThat(recognizer.of(newInputFile("abc.php"))).isNull();
-    assertThat(recognizer.of(newInputFile("abc.java"))).isEqualTo("java");
-    assertThat(recognizer.of(newInputFile("src/abc.java"))).isEqualTo("java");
-    recognizer.stop();
-  }
-
-  @Test
-  public void fail_if_invalid_language() throws Exception {
-    Languages languages = new Languages(new MockLanguage("java", "java"), new MockLanguage("php", "php"));
-
-    Settings settings = new Settings();
-    settings.setProperty(CoreProperties.PROJECT_LANGUAGE_PROPERTY, "unknow");
-    LanguageRecognizer recognizer = new LanguageRecognizer(settings, languages);
-    recognizer.start();
-    thrown.expect(SonarException.class);
-    thrown.expectMessage("No language is installed with key 'unknow'. Please update property 'sonar.language'");
-    recognizer.of(newInputFile("abc"));
-    recognizer.stop();
-  }
-
-  @Test
-  public void fail_if_conflicting_language_suffix() throws Exception {
-    Languages languages = new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml"));
-
-    Settings settings = new Settings();
-    LanguageRecognizer recognizer = new LanguageRecognizer(settings, languages);
-    recognizer.start();
-    thrown.expect(SonarException.class);
-    thrown.expectMessage(new BaseMatcher<String>() {
-      @Override
-      public void describeTo(Description arg0) {
-      }
-
-      @Override
-      public boolean matches(Object arg0) {
-        // Need custom matcher because order of language in the exception is not deterministic (hashmap)
-        return arg0.toString().contains("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ")
-          && arg0.toString().contains("sonar.lang.patterns.web : **/*.xhtml")
-          && arg0.toString().contains("sonar.lang.patterns.xml : **/*.xhtml");
-      }
-    });
-    recognizer.of(newInputFile("abc.xhtml"));
-    recognizer.stop();
-  }
-
-  @Test
-  public void solve_conflict_using_filepattern() throws Exception {
-    Languages languages = new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml"));
-
-    Settings settings = new Settings();
-    settings.setProperty("sonar.lang.patterns.xml", "xml/**");
-    settings.setProperty("sonar.lang.patterns.web", "web/**");
-    LanguageRecognizer recognizer = new LanguageRecognizer(settings, languages);
-    recognizer.start();
-    assertThat(recognizer.of(newInputFile("xml/abc.xhtml"))).isEqualTo("xml");
-    assertThat(recognizer.of(newInputFile("web/abc.xhtml"))).isEqualTo("web");
-    recognizer.stop();
-  }
-
-  @Test
-  public void fail_if_conflicting_filepattern() throws Exception {
-    Languages languages = new Languages(new MockLanguage("abap", "abap"), new MockLanguage("cobol", "cobol"));
-
-    Settings settings = new Settings();
-    settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt");
-    settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt");
-    LanguageRecognizer recognizer = new LanguageRecognizer(settings, languages);
-    recognizer.start();
-    assertThat(recognizer.of(newInputFile("abc.abap"))).isEqualTo("abap");
-    assertThat(recognizer.of(newInputFile("abc.cobol"))).isEqualTo("cobol");
-    thrown.expect(SonarException.class);
-    thrown.expectMessage(new BaseMatcher<String>() {
-      @Override
-      public void describeTo(Description arg0) {
-      }
-
-      @Override
-      public boolean matches(Object arg0) {
-        // Need custom matcher because order of language in the exception is not deterministic (hashmap)
-        return arg0.toString().contains("Language of file 'abc.txt' can not be decided as the file matches patterns of both ")
-          && arg0.toString().contains("sonar.lang.patterns.abap : *.abap,*.txt")
-          && arg0.toString().contains("sonar.lang.patterns.cobol : *.cobol,*.txt");
-      }
-    });
-    recognizer.of(newInputFile("abc.txt"));
-    recognizer.stop();
-  }
-
-  private InputFile newInputFile(String path) throws IOException {
-    File basedir = temp.newFolder();
-    return new InputFileBuilder(new File(basedir, path), Charsets.UTF_8, path).build();
-  }
-
-  static class MockLanguage implements Language {
-    private final String key;
-    private final String[] extensions;
-
-    MockLanguage(String key, String... extensions) {
-      this.key = key;
-      this.extensions = extensions;
-    }
-
-    @Override
-    public String getKey() {
-      return key;
-    }
-
-    @Override
-    public String getName() {
-      return key;
-    }
-
-    @Override
-    public String[] getFileSuffixes() {
-      return extensions;
-    }
-  }
-}

sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/RemoteFileHashesTest.java → sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/PreviousFileHashesLoaderTest.java

@@ -32,12 +32,13 @@ import org.sonar.core.source.db.SnapshotDataDto;
 
 import java.util.Arrays;
 import java.util.Date;
+import java.util.Map;
 
 import static org.fest.assertions.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class RemoteFileHashesTest {
+public class PreviousFileHashesLoaderTest {
 
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
@@ -48,15 +49,14 @@ public class RemoteFileHashesTest {
   PastSnapshotFinder pastSnapshotFinder = mock(PastSnapshotFinder.class);
   Snapshot snapshot = mock(Snapshot.class);
   SnapshotDataDao snapshotDataDao = mock(SnapshotDataDao.class);
-  RemoteFileHashes hashes = new RemoteFileHashes(snapshot, snapshotDataDao, pastSnapshotFinder);
+  PreviousFileHashLoader loader = new PreviousFileHashLoader(snapshot, snapshotDataDao, pastSnapshotFinder);
 
   @Test
-  public void should_return_null_if_no_remote_snapshot() throws Exception {
+  public void should_return_null_if_no_previous_snapshot() throws Exception {
     when(pastSnapshotFinder.findPreviousAnalysis(snapshot)).thenReturn(new PastSnapshot("foo"));
 
-    hashes.start();
-    assertThat(hashes.remoteHash("src/main/java/foo/Bar.java")).isNull();
-    hashes.stop();
+    Map<String, String> hashByRelativePath = loader.hashByRelativePath();
+    assertThat(hashByRelativePath.get("src/main/java/foo/Bar.java")).isNull();
   }
 
   @Test
@@ -65,9 +65,8 @@ public class RemoteFileHashesTest {
     PastSnapshot pastSnapshot = new PastSnapshot("foo", new Date(), previousSnapshot);
     when(pastSnapshotFinder.findPreviousAnalysis(snapshot)).thenReturn(pastSnapshot);
 
-    hashes.start();
-    assertThat(hashes.remoteHash("src/main/java/foo/Bar.java")).isNull();
-    hashes.stop();
+    Map<String, String> hashByRelativePath = loader.hashByRelativePath();
+    assertThat(hashByRelativePath.get("src/main/java/foo/Bar.java")).isNull();
   }
 
   @Test
@@ -82,8 +81,7 @@ public class RemoteFileHashesTest {
     when(snapshotDataDao.selectSnapshotData(123, Arrays.asList(SnapshotDataTypes.FILE_HASHES)))
       .thenReturn(Arrays.asList(snapshotDataDto));
 
-    hashes.start();
-    assertThat(hashes.remoteHash("src/main/java/foo/Bar.java")).isEqualTo("abcd1234");
-    hashes.stop();
+    Map<String, String> hashByRelativePath = loader.hashByRelativePath();
+    assertThat(hashByRelativePath.get("src/main/java/foo/Bar.java")).isEqualTo("abcd1234");
   }
 }

sonar-plugin-api/src/main/java/org/sonar/api/scan/filesystem/internal/DefaultInputDir.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.api.scan.filesystem.internal;
 
+import com.google.common.collect.Maps;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.scan.filesystem.InputDir;
@@ -51,6 +52,12 @@ public class DefaultInputDir implements InputDir {
     this.attributes = attributes;
   }
 
+  public DefaultInputDir(String absolutePath, String path) {
+    this.absolutePath = absolutePath;
+    this.path = path;
+    this.attributes = Maps.newHashMap();
+  }
+
   /**
    * Plugins must not build their own instances of {@link InputDir}.
    * {@link org.sonar.api.scan.filesystem.ModuleFileSystem} must be used to search for inputDir.
@@ -116,4 +123,9 @@ public class DefaultInputDir implements InputDir {
   public String toString() {
     return String.format("[%s]", path);
   }
+
+  public DefaultInputDir setKey(String s) {
+    attributes.put(ATTRIBUTE_COMPONENT_KEY, s);
+    return this;
+  }
 }

sonar-plugin-api/src/main/java/org/sonar/api/scan/filesystem/internal/DefaultInputFile.java

@@ -26,7 +26,6 @@ import org.sonar.api.scan.filesystem.InputFile;
 import org.sonar.api.utils.PathUtils;
 
 import javax.annotation.CheckForNull;
-
 import java.io.File;
 import java.nio.charset.Charset;
 import java.util.Map;
@@ -67,6 +66,7 @@ public class DefaultInputFile implements InputFile {
   private final String path;
   private final Map<String, String> attributes;
   private final String encoding;
+  private long lines = 0L;
 
   private DefaultInputFile(File file, Charset encoding, String path, Map<String, String> attributes) {
     this.encoding = encoding.name();
@@ -148,6 +148,69 @@ public class DefaultInputFile implements InputFile {
     return absolutePath.hashCode();
   }
 
+  public DefaultInputFile setLines(long l) {
+    this.lines = l;
+    return this;
+  }
+
+  public String language() {
+    return attributes.get(ATTRIBUTE_LANGUAGE);
+  }
+
+  public DefaultInputFile setLanguage(String s) {
+    attributes.put(ATTRIBUTE_LANGUAGE, s);
+    return this;
+  }
+
+  public DefaultInputFile setHash(String s) {
+    attributes.put(ATTRIBUTE_HASH, s);
+    return this;
+  }
+
+  public DefaultInputFile setStatus(String s) {
+    attributes.put(ATTRIBUTE_STATUS, s);
+    return this;
+  }
+
+  public DefaultInputFile setKey(String s) {
+    attributes.put(ATTRIBUTE_COMPONENT_KEY, s);
+    return this;
+  }
+
+  public DefaultInputFile setDeprecatedKey(String s) {
+    attributes.put(ATTRIBUTE_COMPONENT_DEPRECATED_KEY, s);
+    return this;
+  }
+
+  public DefaultInputFile setType(String s) {
+    attributes.put(ATTRIBUTE_TYPE, s);
+    return this;
+  }
+
+  /**
+   * Used only for backward-compatibility. Meaningless since version 4.2.
+   */
+  public String sourceDirAbsolutePath() {
+    return attributes.get(ATTRIBUTE_SOURCEDIR_PATH);
+  }
+
+  public DefaultInputFile setSourceDirAbsolutePath(String s) {
+    attributes.put(ATTRIBUTE_SOURCEDIR_PATH, FilenameUtils.normalize(s, true));
+    return this;
+  }
+
+  /**
+   * Used only for backward-compatibility. Meaningless since version 4.2.
+   */
+  public String pathRelativeToSourceDir() {
+    return attributes.get(ATTRIBUTE_SOURCE_RELATIVE_PATH);
+  }
+
+  public DefaultInputFile setPathRelativeToSourceDir(String s) {
+    attributes.put(ATTRIBUTE_SOURCE_RELATIVE_PATH, FilenameUtils.normalize(s, true));
+    return this;
+  }
+
   @Override
   public String toString() {
     return String.format("[%s,%s]", path, type());