SONAR-5868 Improve performances of fs indexation

7 files changed, 104 insertions, 95 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
index 1eda06b2684..e6955102a58 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
@@ -219,8 +219,7 @@ public class DefaultModuleFileSystem extends DefaultFileSystem implements Module
   @Override
   protected void doPreloadFiles() {
     if (!initialized) {
-      LOG.warn("Accessing the filesystem before the Sensor phase is deprecated and will not be supported in the future. Please update your plugin.");
-      indexer.index(this);
+      throw MessageException.of("Accessing the filesystem before the Sensor phase is not supported. Please update your plugin.");
     }
   }
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndexer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndexer.java
index 722e5600a87..4e424011f28 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndexer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndexer.java
@@ -19,7 +19,6 @@
  */
 package org.sonar.batch.scan.filesystem;
 
-import com.google.common.collect.Sets;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.apache.commons.io.filefilter.HiddenFileFilter;
@@ -87,7 +86,7 @@ public class FileIndexer implements BatchComponent {
     LOG.info("Index files");
     exclusionFilters.prepare();
 
-    Progress progress = new Progress(fileCache.filesByModule(fileSystem.moduleKey()), fileCache.dirsByModule(fileSystem.moduleKey()));
+    Progress progress = new Progress();
 
     InputFileBuilder inputFileBuilder = inputFileBuilderFactory.create(fileSystem);
     indexFiles(fileSystem, progress, inputFileBuilder, fileSystem.sources(), InputFile.Type.MAIN);
@@ -103,14 +102,6 @@ public class FileIndexer implements BatchComponent {
       fileSystem.add(indexed);
     }
 
-    // Remove paths that have been removed since previous indexation
-    for (InputFile removed : progress.removed) {
-      fileCache.remove(fileSystem.moduleKey(), removed);
-    }
-    for (InputDir removed : progress.removedDir) {
-      fileCache.remove(fileSystem.moduleKey(), removed);
-    }
-
     LOG.info(String.format("%d files indexed", progress.count()));
 
   }
@@ -194,15 +185,11 @@ public class FileIndexer implements BatchComponent {
   }
 
   private static class Progress {
-    private final Set<InputFile> removed;
-    private final Set<InputDir> removedDir;
     private final Set<InputFile> indexed;
     private final Set<InputDir> indexedDir;
     private final List<Callable<Void>> indexingTasks;
 
-    Progress(Iterable<InputFile> removed, Iterable<InputDir> removedDir) {
-      this.removed = Sets.newHashSet(removed);
-      this.removedDir = Sets.newHashSet(removedDir);
+    Progress() {
       this.indexed = new HashSet<InputFile>();
       this.indexedDir = new HashSet<InputDir>();
       this.indexingTasks = new ArrayList<Callable<Void>>();
@@ -217,12 +204,10 @@ public class FileIndexer implements BatchComponent {
         throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce "
           + "disjoint sets for main and test files");
       }
-      removed.remove(inputFile);
       indexed.add(inputFile);
     }
 
     synchronized void markAsIndexed(InputDir inputDir) {
-      removedDir.remove(inputDir);
       indexedDir.add(inputDir);
     }
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java
index 7fdc0f90033..161497f37ef 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java
@@ -19,11 +19,11 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import com.google.common.base.Charsets;
 import com.google.common.primitives.Longs;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -31,6 +31,8 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
 import java.nio.charset.Charset;
 import java.security.MessageDigest;
 import java.util.ArrayList;
@@ -45,14 +47,6 @@ class FileMetadata {
   private static final char LINE_FEED = '\n';
   private static final char CARRIAGE_RETURN = '\r';
   private static final char BOM = '\uFEFF';
-  private static final String SPACE_CHARS = "\t\n\r ";
-
-  // This singleton aims only to increase the coverage by allowing
-  // to test the private method !
-  static final FileMetadata INSTANCE = new FileMetadata();
-
-  private FileMetadata() {
-  }
 
   /**
    * Compute hash of a file ignoring line ends differences.
@@ -63,17 +57,17 @@ class FileMetadata {
     long currentOriginalOffset = 0;
     List<Long> originalLineOffsets = new ArrayList<Long>();
     List<Object> lineHashes = new ArrayList<Object>();
-    StringBuilder currentLineStr = new StringBuilder();
     int lines = 0;
     char c = (char) -1;
     try {
       MessageDigest globalMd5Digest = DigestUtils.getMd5Digest();
-      globalMd5Digest.reset();
+      MessageDigest lineMd5Digest = DigestUtils.getMd5Digest();
       reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), encoding));
       int i = reader.read();
       boolean afterCR = false;
       // First offset of first line is always 0
       originalLineOffsets.add(0L);
+      boolean blankline = true;
       while (i != -1) {
         c = (char) i;
         if (c == BOM) {
@@ -98,18 +92,21 @@ class FileMetadata {
         if (c == LINE_FEED) {
           lines++;
           originalLineOffsets.add(currentOriginalOffset);
-          lineHashes.add(md5IgnoreWhitespace(currentLineStr));
-          currentLineStr.setLength(0);
+          lineHashes.add(blankline ? null : lineMd5Digest.digest());
+          blankline = true;
         } else {
-          currentLineStr.append(c);
+          if (!Character.isWhitespace(c)) {
+            blankline = false;
+            updateDigestUTF8Char(c, lineMd5Digest);
+          }
         }
-        globalMd5Digest.update(charToBytesUTF(c));
+        updateDigestUTF8Char(c, globalMd5Digest);
         i = reader.read();
       }
       if (c != (char) -1) {
-        // Last empty line
+        // Last line
         lines++;
-        lineHashes.add(md5IgnoreWhitespace(currentLineStr));
+        lineHashes.add(blankline ? null : lineMd5Digest.digest());
       }
       String filehash = Hex.encodeHexString(globalMd5Digest.digest());
       return new Metadata(lines, filehash, originalLineOffsets, lineHashes.toArray(new byte[0][]));
@@ -121,23 +118,17 @@ class FileMetadata {
     }
   }
 
-  private byte[] md5IgnoreWhitespace(StringBuilder currentLineStr) {
-    String reducedLine = StringUtils.replaceChars(currentLineStr.toString(), SPACE_CHARS, "");
-    if (reducedLine.isEmpty()) {
-      return null;
-    }
-    return DigestUtils.md5(reducedLine);
-  }
-
-  private byte[] charToBytesUTF(char c) {
-    char[] buffer = new char[] {c};
-    byte[] b = new byte[buffer.length << 1];
-    for (int i = 0; i < buffer.length; i++) {
-      int bpos = i << 1;
-      b[bpos] = (byte) ((buffer[i] & 0xFF00) >> 8);
-      b[bpos + 1] = (byte) (buffer[i] & 0x00FF);
+  private void updateDigestUTF8Char(char c, MessageDigest md5Digest) {
+    CharBuffer cb = CharBuffer.allocate(1);
+    cb.put(c);
+    cb.flip();
+    ByteBuffer bb = Charsets.UTF_8.encode(cb);
+    byte[] array = bb.array();
+    for (int i = 0; i < array.length; i++) {
+      if (array[i] != 0) {
+        md5Digest.update(array[i]);
+      }
     }
-    return b;
   }
 
   static class Metadata {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
index b961e072890..8b00b2640a4 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
@@ -95,8 +95,15 @@ class InputFileBuilder {
   DeprecatedDefaultInputFile complete(DeprecatedDefaultInputFile inputFile, InputFile.Type type) {
     inputFile.setType(type);
     inputFile.setBasedir(fs.baseDir());
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(inputFile.file(), fs.encoding());
     inputFile.setEncoding(fs.encoding().name());
+
+    String lang = langDetection.language(inputFile);
+    if (lang == null) {
+      return null;
+    }
+    inputFile.setLanguage(lang);
+
+    FileMetadata.Metadata metadata = new FileMetadata().read(inputFile.file(), fs.encoding());
     inputFile.setLines(metadata.lines);
     inputFile.setHash(metadata.hash);
     inputFile.setOriginalLineOffsets(metadata.originalLineOffsets);
@@ -105,11 +112,6 @@ class InputFileBuilder {
     if (analysisMode.isIncremental() && inputFile.status() == InputFile.Status.SAME) {
       return null;
     }
-    String lang = langDetection.language(inputFile);
-    if (lang == null) {
-      return null;
-    }
-    inputFile.setLanguage(lang);
     fillDeprecatedData(inputFile);
     return inputFile;
   }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java
index 4e5822d0827..bdd964621b4 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java
@@ -21,6 +21,7 @@ package org.sonar.batch.mediumtest.fs;
 
 import com.google.common.collect.ImmutableMap;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -32,7 +33,6 @@ import org.sonar.api.utils.MessageException;
 import org.sonar.api.utils.System2;
 import org.sonar.batch.mediumtest.BatchMediumTester;
 import org.sonar.batch.mediumtest.BatchMediumTester.TaskResult;
-import org.sonar.batch.protocol.input.ActiveRule;
 import org.sonar.xoo.XooPlugin;
 
 import java.io.File;
@@ -51,7 +51,6 @@ public class FileSystemMediumTest {
   public BatchMediumTester tester = BatchMediumTester.builder()
     .registerPlugin("xoo", new XooPlugin())
     .addDefaultQProfile("xoo", "Sonar Way")
-    .activateRule(new ActiveRule("xoo", "OneIssuePerLine", "One issue per line", "MAJOR", "xoo", "xoo"))
     .bootstrapProperties(ImmutableMap.of("sonar.analysis.mode", "sensor"))
     .build();
 
@@ -101,6 +100,28 @@ public class FileSystemMediumTest {
   }
 
   @Test
+  public void scanBigProject() throws IOException {
+    File srcDir = new File(baseDir, "src");
+    srcDir.mkdir();
+
+    int nbFiles = 100;
+    int ruleCount = 100000;
+    for (int nb = 1; nb <= nbFiles; nb++) {
+      File xooFile = new File(srcDir, "sample" + nb + ".xoo");
+      FileUtils.write(xooFile, StringUtils.repeat(StringUtils.repeat("a", 100) + "\n", ruleCount / 1000));
+    }
+
+    TaskResult result = tester.newTask()
+      .properties(builder
+        .put("sonar.sources", "src")
+        .build())
+      .start();
+
+    assertThat(result.inputFiles()).hasSize(100);
+    assertThat(result.inputDirs()).hasSize(1);
+  }
+
+  @Test
   public void scanProjectWithTestDir() throws IOException {
     File test = new File(baseDir, "test");
     test.mkdir();
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java
index 72a415020f1..4a4822f5a70 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java
@@ -20,7 +20,6 @@
 package org.sonar.batch.scan.filesystem;
 
 import com.google.common.base.Charsets;
-import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.junit.Rule;
 import org.junit.Test;
@@ -29,15 +28,12 @@ import org.junit.rules.TemporaryFolder;
 
 import java.io.File;
 
+import static org.apache.commons.codec.digest.DigestUtils.md5;
+import static org.apache.commons.codec.digest.DigestUtils.md5Hex;
 import static org.fest.assertions.Assertions.assertThat;
 
 public class FileMetadataTest {
 
-  private static final String EXPECTED_HASH_WITHOUT_LATEST_EOL = "c80cc50d65ace6c4eb63f189d274dbeb";
-  private static final String EXPECTED_HASH_NEW_LINE_FIRST = "cf2d41454b5b451eeb5122f0848c1d2a";
-  private static final String EXPECTED_HASH_WITH_LATEST_EOL = "bf77e51d219e7d7d643faac86f1b5d15";
-  private static final String NON_ASCII = "4050369e8ba432c9079e258b43fe4ab5";
-
   @Rule
   public ExpectedException thrown = ExpectedException.none();
 
@@ -49,7 +45,7 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.touch(tempFile);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(0);
     assertThat(metadata.hash).isNotEmpty();
     assertThat(metadata.originalLineOffsets).containsOnly(0);
@@ -61,9 +57,9 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "foo\r\nbar\r\nbaz", Charsets.UTF_8, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(3);
-    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz"));
     assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10);
     assertThat(metadata.lineHashes[0]).containsOnly(md5("foo"));
     assertThat(metadata.lineHashes[1]).containsOnly(md5("bar"));
@@ -75,9 +71,9 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "föo\r\nbàr\r\n\u1D11Ebaßz\r\n", Charsets.UTF_8, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(4);
-    assertThat(metadata.hash).isEqualTo(NON_ASCII);
+    assertThat(metadata.hash).isEqualTo(md5Hex("föo\nbàr\n\u1D11Ebaßz\n"));
     assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18);
     assertThat(metadata.lineHashes[0]).containsOnly(md5("föo"));
     assertThat(metadata.lineHashes[1]).containsOnly(md5("bàr"));
@@ -90,9 +86,9 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "föo\r\nbàr\r\n\u1D11Ebaßz\r\n", Charsets.UTF_16, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_16);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_16);
     assertThat(metadata.lines).isEqualTo(4);
-    assertThat(metadata.hash).isEqualTo(NON_ASCII);
+    assertThat(metadata.hash).isEqualTo(md5Hex("föo\nbàr\n\u1D11Ebaßz\n"));
     assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18);
     assertThat(metadata.lineHashes[0]).containsOnly(md5("föo"));
     assertThat(metadata.lineHashes[1]).containsOnly(md5("bàr"));
@@ -105,9 +101,9 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "foo\nbar\nbaz", Charsets.UTF_8, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(3);
-    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz"));
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8);
     assertThat(metadata.lineHashes[0]).containsOnly(md5("foo"));
     assertThat(metadata.lineHashes[1]).containsOnly(md5("bar"));
@@ -119,9 +115,9 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "foo\nbar\nbaz\n", Charsets.UTF_8, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(4);
-    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz\n"));
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8, 12);
     assertThat(metadata.lineHashes[0]).containsOnly(md5("foo"));
     assertThat(metadata.lineHashes[1]).containsOnly(md5("bar"));
@@ -134,9 +130,9 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "foo\nbar\r\nbaz\n", Charsets.UTF_8, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(4);
-    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz\n"));
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9, 13);
     assertThat(metadata.lineHashes[0]).containsOnly(md5("foo"));
     assertThat(metadata.lineHashes[1]).containsOnly(md5("bar"));
@@ -145,13 +141,28 @@ public class FileMetadataTest {
   }
 
   @Test
+  public void several_new_lines() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\n\n\nbar", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(4);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\n\n\nbar"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 5, 6);
+    assertThat(metadata.lineHashes[0]).containsOnly(md5("foo"));
+    assertThat(metadata.lineHashes[1]).isNull();
+    assertThat(metadata.lineHashes[2]).isNull();
+    assertThat(metadata.lineHashes[3]).containsOnly(md5("bar"));
+  }
+
+  @Test
   public void mix_of_newlines_without_latest_eol() throws Exception {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "foo\nbar\r\nbaz", Charsets.UTF_8, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(3);
-    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz"));
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9);
     assertThat(metadata.lineHashes[0]).containsOnly(md5("foo"));
     assertThat(metadata.lineHashes[1]).containsOnly(md5("bar"));
@@ -163,9 +174,9 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "\nfoo\nbar\r\nbaz", Charsets.UTF_8, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(4);
-    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_NEW_LINE_FIRST);
+    assertThat(metadata.hash).isEqualTo(md5Hex("\nfoo\nbar\nbaz"));
     assertThat(metadata.originalLineOffsets).containsOnly(0, 1, 5, 10);
     assertThat(metadata.lineHashes[0]).isNull();
     assertThat(metadata.lineHashes[1]).containsOnly(md5("foo"));
@@ -178,9 +189,9 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "\uFEFFfoo\nbar\r\nbaz", Charsets.UTF_8, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(3);
-    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz"));
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9);
     assertThat(metadata.lineHashes[0]).containsOnly(md5("foo"));
     assertThat(metadata.lineHashes[1]).containsOnly(md5("bar"));
@@ -192,8 +203,9 @@ public class FileMetadataTest {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, " foo\nb ar\r\nbaz \t", Charsets.UTF_8, true);
 
-    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex(" foo\nb ar\nbaz \t"));
     assertThat(metadata.lineHashes[0]).containsOnly(md5("foo"));
     assertThat(metadata.lineHashes[1]).containsOnly(md5("bar"));
     assertThat(metadata.lineHashes[2]).containsOnly(md5("baz"));
@@ -207,7 +219,7 @@ public class FileMetadataTest {
     thrown.expect(IllegalStateException.class);
     thrown.expectMessage("Fail to read file '" + file.getAbsolutePath() + "' with encoding 'UTF-8'");
 
-    FileMetadata.INSTANCE.read(file, Charsets.UTF_8);
+    new FileMetadata().read(file, Charsets.UTF_8);
   }
 
   @Test
@@ -222,14 +234,11 @@ public class FileMetadataTest {
     File file2 = temp.newFile();
     FileUtils.write(file2, "foo\nbar", Charsets.UTF_8, true);
 
-    String hash1 = FileMetadata.INSTANCE.read(file1, Charsets.UTF_8).hash;
-    String hash1a = FileMetadata.INSTANCE.read(file1a, Charsets.UTF_8).hash;
-    String hash2 = FileMetadata.INSTANCE.read(file2, Charsets.UTF_8).hash;
+    String hash1 = new FileMetadata().read(file1, Charsets.UTF_8).hash;
+    String hash1a = new FileMetadata().read(file1a, Charsets.UTF_8).hash;
+    String hash2 = new FileMetadata().read(file2, Charsets.UTF_8).hash;
     assertThat(hash1).isEqualTo(hash1a);
     assertThat(hash1).isNotEqualTo(hash2);
   }
 
-  private static byte[] md5(String input) {
-    return DigestUtils.md5(input);
-  }
 }
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultFileSystem.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultFileSystem.java
index 0e67c3cf9ac..6da2aa44811 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultFileSystem.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultFileSystem.java
@@ -181,8 +181,9 @@ public class DefaultFileSystem implements FileSystem {
 
   /**
    * Adds InputFile to the list and registers its language, if present.
+   * Synchronized because PersistIt Exchange is not concurrent
    */
-  public DefaultFileSystem add(InputFile inputFile) {
+  public synchronized DefaultFileSystem add(InputFile inputFile) {
     cache.add(inputFile);
     if (inputFile.language() != null) {
       languages.add(inputFile.language());
@@ -192,8 +193,9 @@ public class DefaultFileSystem implements FileSystem {
 
   /**
    * Adds InputDir to the list.
+   * Synchronized because PersistIt Exchange is not concurrent
    */
-  public DefaultFileSystem add(InputDir inputDir) {
+  public synchronized DefaultFileSystem add(InputDir inputDir) {
     cache.add(inputDir);
     return this;
   }