SONAR-5827 Feed the new "file_sources" table

16 files changed, 453 insertions, 17 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/highlighting/SyntaxHighlightingData.java b/sonar-batch/src/main/java/org/sonar/batch/highlighting/SyntaxHighlightingData.java
index 2a539efe189..65c1633a039 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/highlighting/SyntaxHighlightingData.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/highlighting/SyntaxHighlightingData.java
@@ -23,19 +23,20 @@ import org.sonar.batch.index.Data;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 public class SyntaxHighlightingData implements Data {
 
-  private static final String FIELD_SEPARATOR = ",";
-  private static final String RULE_SEPARATOR = ";";
+  public static final String FIELD_SEPARATOR = ",";
+  public static final String RULE_SEPARATOR = ";";
 
-  private Collection<SyntaxHighlightingRule> syntaxHighlightingRuleSet;
+  private List<SyntaxHighlightingRule> syntaxHighlightingRuleSet;
 
   public SyntaxHighlightingData(Collection<SyntaxHighlightingRule> syntaxHighlightingRuleSet) {
     this.syntaxHighlightingRuleSet = new ArrayList<SyntaxHighlightingRule>(syntaxHighlightingRuleSet);
   }
 
-  public Collection<SyntaxHighlightingRule> syntaxHighlightingRuleSet() {
+  public List<SyntaxHighlightingRule> syntaxHighlightingRuleSet() {
     return syntaxHighlightingRuleSet;
   }
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
index 42940c0af88..925f6919adf 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
@@ -19,23 +19,71 @@
  */
 package org.sonar.batch.index;
 
+import com.google.common.base.CharMatcher;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.FileUtils;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.InputPath;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 import org.sonar.api.database.model.Snapshot;
+import org.sonar.api.measures.CoreMetrics;
+import org.sonar.api.measures.Measure;
 import org.sonar.api.resources.Resource;
+import org.sonar.api.utils.KeyValueFormat;
+import org.sonar.api.utils.System2;
+import org.sonar.api.utils.text.CsvWriter;
+import org.sonar.batch.ProjectTree;
+import org.sonar.batch.highlighting.SyntaxHighlightingData;
+import org.sonar.batch.highlighting.SyntaxHighlightingRule;
+import org.sonar.batch.scan.filesystem.InputPathCache;
+import org.sonar.batch.scan.measure.MeasureCache;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.core.persistence.MyBatis;
+import org.sonar.core.source.SnapshotDataTypes;
+import org.sonar.core.source.db.FileSourceDto;
+import org.sonar.core.source.db.FileSourceMapper;
 import org.sonar.core.source.db.SnapshotSourceDao;
 import org.sonar.core.source.db.SnapshotSourceDto;
 
 import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.Collections;
 import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 
-public class SourcePersister {
+import static com.google.common.base.Charsets.UTF_8;
 
-  private ResourcePersister resourcePersister;
+public class SourcePersister implements ScanPersister {
+
+  private static final String BOM = "\uFEFF";
+  private final ResourcePersister resourcePersister;
   private final SnapshotSourceDao sourceDao;
+  private final InputPathCache inputPathCache;
+  private final MyBatis mybatis;
+  private final MeasureCache measureCache;
+  private final ComponentDataCache componentDataCache;
+  private final System2 system2;
+  private final ProjectTree projectTree;
+  private final ResourceCache resourceCache;
 
-  public SourcePersister(ResourcePersister resourcePersister, SnapshotSourceDao sourceDao) {
+  public SourcePersister(ResourcePersister resourcePersister, SnapshotSourceDao sourceDao, InputPathCache inputPathCache,
+    MyBatis mybatis, MeasureCache measureCache, ComponentDataCache componentDataCache, ProjectTree projectTree, System2 system2, ResourceCache resourceCache) {
     this.resourcePersister = resourcePersister;
     this.sourceDao = sourceDao;
+    this.inputPathCache = inputPathCache;
+    this.mybatis = mybatis;
+    this.measureCache = measureCache;
+    this.componentDataCache = componentDataCache;
+    this.projectTree = projectTree;
+    this.system2 = system2;
+    this.resourceCache = resourceCache;
   }
 
   public void saveSource(Resource resource, String source, Date updatedAt) {
@@ -55,4 +103,133 @@ public class SourcePersister {
     }
     return null;
   }
+
+  @Override
+  public void persist() {
+    DbSession session = mybatis.openSession(false);
+    try {
+      FileSourceMapper mapper = session.getMapper(FileSourceMapper.class);
+
+      for (InputPath inputPath : inputPathCache.all()) {
+        if (inputPath instanceof InputFile) {
+          DefaultInputFile inputFile = (DefaultInputFile) inputPath;
+          org.sonar.api.resources.File file = (org.sonar.api.resources.File) resourceCache.get(inputFile.key());
+          String fileUuid = file.getUuid();
+          FileSourceDto previous = mapper.select(fileUuid);
+          String newData = getSourceData(inputFile);
+          String dataHash = DigestUtils.md5Hex(newData);
+          Date now = system2.newDate();
+          if (previous == null) {
+            FileSourceDto newFileSource = new FileSourceDto().setProjectUuid(projectTree.getRootProject().getUuid()).setFileUuid(fileUuid).setData(newData).setDataHash(dataHash)
+              .setCreatedAt(now)
+              .setUpdatedAt(now);
+            mapper.insert(newFileSource);
+          } else {
+            if (dataHash.equals(previous.getDataHash())) {
+              continue;
+            } else {
+              previous.setData(newData).setDataHash(dataHash).setUpdatedAt(now);
+              mapper.update(previous);
+            }
+          }
+        }
+        session.commit();
+      }
+    } catch (Exception e) {
+      throw new IllegalStateException("Unable to save file sources", e);
+    } finally {
+      MyBatis.closeQuietly(session);
+    }
+
+  }
+
+  String getSourceData(DefaultInputFile file) {
+    if (file.lines() == 0) {
+      return "";
+    }
+    List<String> lines;
+    try {
+      lines = FileUtils.readLines(file.file(), file.encoding());
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to read file", e);
+    }
+    // Missing empty last line
+    if (lines.size() == file.lines() - 1) {
+      lines.add("");
+    }
+    Map<Integer, String> authorsByLine = getLineMetric(file, CoreMetrics.SCM_AUTHORS_BY_LINE_KEY);
+    Map<Integer, String> revisionsByLine = getLineMetric(file, CoreMetrics.SCM_REVISIONS_BY_LINE_KEY);
+    Map<Integer, String> datesByLine = getLineMetric(file, CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE_KEY);
+    SyntaxHighlightingData highlighting = componentDataCache.getData(file.key(), SnapshotDataTypes.SYNTAX_HIGHLIGHTING);
+    String[] highlightingPerLine = computeHighlightingPerLine(file, highlighting);
+
+    ByteArrayOutputStream output = new ByteArrayOutputStream();
+    CsvWriter csv = CsvWriter.of(new OutputStreamWriter(output, UTF_8));
+    for (int lineIdx = 1; lineIdx <= file.lines(); lineIdx++) {
+      csv.values(revisionsByLine.get(lineIdx), authorsByLine.get(lineIdx), datesByLine.get(lineIdx), highlightingPerLine[lineIdx - 1],
+        CharMatcher.anyOf(BOM).removeFrom(lines.get(lineIdx - 1)));
+    }
+    csv.close();
+    return new String(output.toByteArray(), UTF_8);
+  }
+
+  String[] computeHighlightingPerLine(DefaultInputFile file, @Nullable SyntaxHighlightingData highlighting) {
+    String[] highlightingPerLine = new String[file.lines()];
+    if (highlighting == null) {
+      return highlightingPerLine;
+    }
+    Iterable<SyntaxHighlightingRule> rules = highlighting.syntaxHighlightingRuleSet();
+    int currentLineIdx = 1;
+    StringBuilder currentLineSb = new StringBuilder();
+    for (SyntaxHighlightingRule rule : rules) {
+      long ruleStartOffset = rule.getStartPosition();
+      long ruleEndOffset = rule.getEndPosition();
+      while (currentLineIdx < file.lines() && ruleStartOffset >= file.originalLineOffsets()[currentLineIdx]) {
+        // This rule starts on another line so advance
+        saveLineHighlighting(highlightingPerLine, currentLineIdx, currentLineSb);
+        currentLineIdx++;
+      }
+      // Now we know current rule starts on current line
+      long ruleStartOffsetCurrentLine = ruleStartOffset;
+      while (currentLineIdx < file.lines() && ruleEndOffset >= file.originalLineOffsets()[currentLineIdx]) {
+        // rule continue on next line so write current line and continue on next line with same rule
+        writeRule(currentLineSb, ruleStartOffsetCurrentLine - file.originalLineOffsets()[currentLineIdx - 1], file.originalLineOffsets()[currentLineIdx] - 1, rule.getTextType());
+        saveLineHighlighting(highlightingPerLine, currentLineIdx, currentLineSb);
+        currentLineIdx++;
+        ruleStartOffsetCurrentLine = file.originalLineOffsets()[currentLineIdx];
+      }
+      // Rule ends on current line
+      writeRule(currentLineSb, ruleStartOffsetCurrentLine - file.originalLineOffsets()[currentLineIdx - 1], ruleEndOffset - file.originalLineOffsets()[currentLineIdx - 1],
+        rule.getTextType());
+    }
+    saveLineHighlighting(highlightingPerLine, currentLineIdx, currentLineSb);
+    return highlightingPerLine;
+  }
+
+  private void writeRule(StringBuilder currentLineSb, long startLineOffset, long endLineOffset, TypeOfText textType) {
+    if (currentLineSb.length() > 0) {
+      currentLineSb.append(SyntaxHighlightingData.RULE_SEPARATOR);
+    }
+    currentLineSb.append(startLineOffset)
+      .append(SyntaxHighlightingData.FIELD_SEPARATOR)
+      .append(endLineOffset)
+      .append(SyntaxHighlightingData.FIELD_SEPARATOR)
+      .append(textType.cssClass());
+  }
+
+  private void saveLineHighlighting(String[] highlightingPerLine, int currentLineIdx, StringBuilder currentLineSb) {
+    highlightingPerLine[currentLineIdx - 1] = currentLineSb.toString();
+    currentLineSb.setLength(0);
+  }
+
+  private Map<Integer, String> getLineMetric(DefaultInputFile file, String metricKey) {
+    Map<Integer, String> authorsByLine;
+    Iterator<Measure> authorsIt = measureCache.byMetric(file.key(), metricKey).iterator();
+    if (authorsIt.hasNext()) {
+      authorsByLine = KeyValueFormat.parseIntString((String) authorsIt.next().value());
+    } else {
+      authorsByLine = Collections.emptyMap();
+    }
+    return authorsByLine;
+  }
 }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
index a148c03d190..32741b9e69d 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
@@ -27,6 +27,7 @@ import org.sonar.api.batch.SonarIndex;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.InputFile.Status;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
 import org.sonar.api.resources.File;
 import org.sonar.api.resources.Languages;
@@ -71,6 +72,7 @@ public class ComponentIndexer implements BatchComponent {
     migration.migrateIfNeeded(module, fs);
 
     for (InputFile inputFile : fs.inputFiles(fs.predicates().all())) {
+      DefaultInputFile defaultInputFile = (DefaultInputFile) inputFile;
       String languageKey = inputFile.language();
       boolean unitTest = InputFile.Type.TEST == inputFile.type();
       String pathFromSourceDir = ((DeprecatedDefaultInputFile) inputFile).pathRelativeToSourceDir();
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultInputFileValueCoder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultInputFileValueCoder.java
index 42398a58edc..49b19f6fec1 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultInputFileValueCoder.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultInputFileValueCoder.java
@@ -46,6 +46,8 @@ class DefaultInputFileValueCoder implements ValueCoder {
     value.putString(f.status().name());
     putUTFOrNull(value, f.hash());
     value.put(f.lines());
+    putUTFOrNull(value, f.encoding());
+    value.putLongArray(f.originalLineOffsets());
   }
 
   private void putUTFOrNull(Value value, @Nullable String utfOrNull) {
@@ -70,6 +72,8 @@ class DefaultInputFileValueCoder implements ValueCoder {
     file.setStatus(InputFile.Status.valueOf(value.getString()));
     file.setHash(value.getString());
     file.setLines(value.getInt());
+    file.setEncoding(value.getString());
+    file.setOriginalLineOffsets(value.getLongArray());
     return file;
   }
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndexer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndexer.java
index f83cf29d5ef..722e5600a87 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndexer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndexer.java
@@ -32,6 +32,7 @@ import org.sonar.api.batch.fs.InputDir;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.InputFileFilter;
 import org.sonar.api.batch.fs.internal.DefaultInputDir;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.api.utils.MessageException;
@@ -165,7 +166,7 @@ public class FileIndexer implements BatchComponent {
 
       @Override
       public Void call() throws Exception {
-        InputFile completedFile = inputFileBuilder.complete(inputFile, type);
+        DefaultInputFile completedFile = inputFileBuilder.complete(inputFile, type);
         if (completedFile != null && accept(completedFile)) {
           status.markAsIndexed(inputFile);
           File parentDir = inputFile.file().getParentFile();
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java
index 4759db79b68..ca6fd9aed4b 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java
@@ -19,6 +19,8 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.IOUtils;
@@ -31,6 +33,8 @@ import java.io.InputStreamReader;
 import java.io.Reader;
 import java.nio.charset.Charset;
 import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Computes hash of files. Ends of Lines are ignored, so files with
@@ -40,6 +44,7 @@ class FileMetadata {
 
   private static final char LINE_FEED = '\n';
   private static final char CARRIAGE_RETURN = '\r';
+  private static final char BOM = '\uFEFF';
 
   // This singleton aims only to increase the coverage by allowing
   // to test the private method !
@@ -54,6 +59,11 @@ class FileMetadata {
    */
   Metadata read(File file, Charset encoding) {
     Reader reader = null;
+    long currentOriginalOffset = 0;
+    List<Long> originalLineOffsets = new ArrayList<Long>();
+    List<Integer> lineCheckSum = new ArrayList<Integer>();
+    int hash = 5381;
+    StringBuilder currentLineStr = new StringBuilder();
     int lines = 0;
     char c = (char) -1;
     try {
@@ -62,11 +72,20 @@ class FileMetadata {
       reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), encoding));
       int i = reader.read();
       boolean afterCR = false;
+      // First offset of first line is always 0
+      originalLineOffsets.add(0L);
       while (i != -1) {
         c = (char) i;
+        if (c == BOM) {
+          // Ignore
+          i = reader.read();
+          continue;
+        }
+        currentOriginalOffset++;
         if (afterCR) {
           afterCR = false;
           if (c == LINE_FEED) {
+            originalLineOffsets.set(originalLineOffsets.size() - 1, originalLineOffsets.get(originalLineOffsets.size() - 1) + 1);
             // Ignore
             i = reader.read();
             continue;
@@ -76,17 +95,24 @@ class FileMetadata {
           afterCR = true;
           c = LINE_FEED;
         }
+        currentLineStr.append(c);
+        hash = ((hash << 5) + hash) + (c & 0xff);
         if (c == LINE_FEED) {
           lines++;
+          originalLineOffsets.add(currentOriginalOffset);
+          lineCheckSum.add(hash);
+          hash = 5381;
+          currentLineStr.setLength(0);
         }
         md5Digest.update(charToBytesUTF(c));
         i = reader.read();
       }
       if (c != (char) -1) {
         lines++;
+        lineCheckSum.add(hash);
       }
-      String hash = Hex.encodeHexString(md5Digest.digest());
-      return new Metadata(lines, hash);
+      String filehash = Hex.encodeHexString(md5Digest.digest());
+      return new Metadata(lines, filehash, originalLineOffsets, lineCheckSum);
 
     } catch (IOException e) {
       throw new IllegalStateException(String.format("Fail to read file '%s' with encoding '%s'", file.getAbsolutePath(), encoding), e);
@@ -107,12 +133,16 @@ class FileMetadata {
   }
 
   static class Metadata {
-    int lines;
-    String hash;
+    final int lines;
+    final String hash;
+    final long[] originalLineOffsets;
+    final int[] lineChecksum;
 
-    private Metadata(int lines, String hash) {
+    private Metadata(int lines, String hash, List<Long> originalLineOffsets, List<Integer> lineCheckSum) {
       this.lines = lines;
       this.hash = hash;
+      this.originalLineOffsets = Longs.toArray(originalLineOffsets);
+      this.lineChecksum = Ints.toArray(lineCheckSum);
     }
   }
 }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
index cc099ba9f9c..a94f059e85a 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
@@ -96,8 +96,10 @@ class InputFileBuilder {
     inputFile.setType(type);
     inputFile.setBasedir(fs.baseDir());
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(inputFile.file(), fs.encoding());
+    inputFile.setEncoding(fs.encoding().name());
     inputFile.setLines(metadata.lines);
     inputFile.setHash(metadata.hash);
+    inputFile.setOriginalLineOffsets(metadata.originalLineOffsets);
     inputFile.setStatus(statusDetection.status(inputFile.moduleKey(), inputFile.relativePath(), metadata.hash));
     if (analysisMode.isIncremental() && inputFile.status() == InputFile.Status.SAME) {
       return null;
diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
index 166d5de0af1..81e96a2627e 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
@@ -19,37 +19,190 @@
  */
 package org.sonar.batch.index;
 
+import org.apache.commons.io.FileUtils;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.InputPath;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 import org.sonar.api.database.model.Snapshot;
+import org.sonar.api.measures.CoreMetrics;
+import org.sonar.api.measures.Measure;
 import org.sonar.api.resources.File;
+import org.sonar.api.resources.Project;
 import org.sonar.api.resources.Resource;
 import org.sonar.api.utils.DateUtils;
+import org.sonar.api.utils.System2;
+import org.sonar.batch.ProjectTree;
+import org.sonar.batch.highlighting.SyntaxHighlightingData;
+import org.sonar.batch.highlighting.SyntaxHighlightingRule;
+import org.sonar.batch.scan.filesystem.InputPathCache;
+import org.sonar.batch.scan.measure.MeasureCache;
 import org.sonar.core.persistence.AbstractDaoTestCase;
+import org.sonar.core.source.SnapshotDataTypes;
 import org.sonar.core.source.db.SnapshotSourceDao;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class SourcePersisterTest extends AbstractDaoTestCase {
 
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
   private SourcePersister sourcePersister;
+  private InputPathCache inputPathCache;
+  private ResourceCache resourceCache;
+  private ProjectTree projectTree;
+  private System2 system2;
+  private MeasureCache measureCache;
+  private ComponentDataCache componentDataCache;
+
+  private static final String PROJECT_KEY = "foo";
+
+  private java.io.File basedir;
 
   @Before
-  public void before() {
-    setupData("shared");
+  public void before() throws IOException {
     ResourcePersister resourcePersister = mock(ResourcePersister.class);
     Snapshot snapshot = new Snapshot();
     snapshot.setId(1000);
     when(resourcePersister.getSnapshotOrFail(any(Resource.class))).thenReturn(snapshot);
-    sourcePersister = new SourcePersister(resourcePersister, new SnapshotSourceDao(getMyBatis()));
+    inputPathCache = mock(InputPathCache.class);
+    resourceCache = mock(ResourceCache.class);
+    projectTree = mock(ProjectTree.class);
+    system2 = mock(System2.class);
+    measureCache = mock(MeasureCache.class);
+    when(measureCache.byMetric(anyString(), anyString())).thenReturn(Collections.<org.sonar.api.measures.Measure>emptyList());
+    componentDataCache = mock(ComponentDataCache.class);
+    sourcePersister = new SourcePersister(resourcePersister, new SnapshotSourceDao(getMyBatis()), inputPathCache,
+      getMyBatis(), measureCache, componentDataCache, projectTree, system2,
+      resourceCache);
+    Project project = new Project(PROJECT_KEY);
+    project.setUuid("projectUuid");
+    when(projectTree.getRootProject()).thenReturn(project);
+    basedir = temp.newFolder();
   }
 
   @Test
   public void shouldSaveSource() {
+    setupData("shouldSaveSource");
     sourcePersister.saveSource(new File("org/foo/Bar.java"), "this is the file content", DateUtils.parseDateTime("2014-10-31T16:44:02+0100"));
     checkTables("shouldSaveSource", "snapshot_sources");
   }
 
+  @Test
+  public void testPersistDontTouchUnchanged() throws Exception {
+    setupData("file_sources");
+    when(system2.newDate()).thenReturn(DateUtils.parseDateTime("2014-10-29T16:44:02+0100"));
+
+    String relativePathSame = "src/same.java";
+    java.io.File sameFile = new java.io.File(basedir, relativePathSame);
+    FileUtils.write(sameFile, "unchanged\ncontent");
+    DefaultInputFile inputFileNew = new DefaultInputFile(PROJECT_KEY, relativePathSame).setLines(2).setAbsolutePath(sameFile.getAbsolutePath());
+    when(inputPathCache.all()).thenReturn(Arrays.<InputPath>asList(inputFileNew));
+
+    mockResourceCache(relativePathSame, PROJECT_KEY, "uuidsame");
+
+    sourcePersister.persist();
+    checkTables("testPersistDontTouchUnchanged", "file_sources");
+  }
+
+  @Test
+  public void testPersistUpdateChanged() throws Exception {
+    setupData("file_sources");
+    when(system2.newDate()).thenReturn(DateUtils.parseDateTime("2014-10-29T16:44:02+0100"));
+
+    String relativePathSame = "src/changed.java";
+    java.io.File sameFile = new java.io.File(basedir, relativePathSame);
+    FileUtils.write(sameFile, "changed\ncontent");
+    DefaultInputFile inputFileNew = new DefaultInputFile(PROJECT_KEY, relativePathSame).setLines(2).setAbsolutePath(sameFile.getAbsolutePath());
+    when(inputPathCache.all()).thenReturn(Arrays.<InputPath>asList(inputFileNew));
+
+    mockResourceCache(relativePathSame, PROJECT_KEY, "uuidsame");
+
+    sourcePersister.persist();
+    checkTables("testPersistUpdateChanged", "file_sources");
+  }
+
+  @Test
+  public void testPersistEmptyFile() throws Exception {
+    setupData("file_sources");
+    when(system2.newDate()).thenReturn(DateUtils.parseDateTime("2014-10-29T16:44:02+0100"));
+
+    String relativePathEmpty = "src/empty.java";
+    DefaultInputFile inputFileEmpty = new DefaultInputFile(PROJECT_KEY, relativePathEmpty).setLines(0);
+    when(inputPathCache.all()).thenReturn(Arrays.<InputPath>asList(inputFileEmpty));
+
+    mockResourceCache(relativePathEmpty, PROJECT_KEY, "uuidempty");
+
+    sourcePersister.persist();
+    checkTables("testPersistEmptyFile", "file_sources");
+  }
+
+  @Test
+  public void testPersistNewFileNoScmNoHighlighting() throws Exception {
+    setupData("file_sources");
+    when(system2.newDate()).thenReturn(DateUtils.parseDateTime("2014-10-29T16:44:02+0100"));
+
+    String relativePathNew = "src/new.java";
+    java.io.File newFile = new java.io.File(basedir, relativePathNew);
+    FileUtils.write(newFile, "foo\nbar\nbiz");
+    DefaultInputFile inputFileNew = new DefaultInputFile(PROJECT_KEY, relativePathNew).setLines(3).setAbsolutePath(newFile.getAbsolutePath());
+    when(inputPathCache.all()).thenReturn(Arrays.<InputPath>asList(inputFileNew));
+
+    mockResourceCache(relativePathNew, PROJECT_KEY, "uuidnew");
+
+    sourcePersister.persist();
+    checkTables("testPersistNewFileNoScmNoHighlighting", "file_sources");
+  }
+
+  @Test
+  public void testPersistNewFileWithScmAndHighlighting() throws Exception {
+    setupData("file_sources");
+    when(system2.newDate()).thenReturn(DateUtils.parseDateTime("2014-10-29T16:44:02+0100"));
+
+    String relativePathNew = "src/new.java";
+    java.io.File newFile = new java.io.File(basedir, relativePathNew);
+    FileUtils.write(newFile, "foo\nbar\nbiz");
+    DefaultInputFile inputFileNew = new DefaultInputFile(PROJECT_KEY, relativePathNew)
+      .setLines(3)
+      .setAbsolutePath(newFile.getAbsolutePath())
+      .setOriginalLineOffsets(new long[] {0, 4, 7});
+    when(inputPathCache.all()).thenReturn(Arrays.<InputPath>asList(inputFileNew));
+
+    mockResourceCache(relativePathNew, PROJECT_KEY, "uuidnew");
+
+    when(measureCache.byMetric(PROJECT_KEY + ":" + relativePathNew, CoreMetrics.SCM_AUTHORS_BY_LINE_KEY))
+      .thenReturn(Arrays.asList(new Measure(CoreMetrics.SCM_AUTHORS_BY_LINE, "1=julien;2=simon;3=julien")));
+    when(measureCache.byMetric(PROJECT_KEY + ":" + relativePathNew, CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE_KEY))
+      .thenReturn(Arrays.asList(new Measure(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE, "1=2014-10-11T16:44:02+0100;2=2014-10-12T16:44:02+0100;3=2014-10-13T16:44:02+0100")));
+    when(measureCache.byMetric(PROJECT_KEY + ":" + relativePathNew, CoreMetrics.SCM_REVISIONS_BY_LINE_KEY))
+      .thenReturn(Arrays.asList(new Measure(CoreMetrics.SCM_REVISIONS_BY_LINE, "1=123;2=234;3=345")));
+
+    SyntaxHighlightingData highlighting = new SyntaxHighlightingData(Arrays.asList(
+      SyntaxHighlightingRule.create(0, 3, TypeOfText.ANNOTATION),
+      SyntaxHighlightingRule.create(4, 5, TypeOfText.COMMENT),
+      SyntaxHighlightingRule.create(7, 16, TypeOfText.CONSTANT)
+      ));
+    when(componentDataCache.getData(PROJECT_KEY + ":" + relativePathNew, SnapshotDataTypes.SYNTAX_HIGHLIGHTING))
+      .thenReturn(highlighting);
+
+    sourcePersister.persist();
+    checkTables("testPersistNewFileWithScmAndHighlighting", "file_sources");
+  }
+
+  private void mockResourceCache(String relativePathEmpty, String projectKey, String uuid) {
+    File sonarFile = File.create(relativePathEmpty);
+    sonarFile.setUuid(uuid);
+    when(resourceCache.get(projectKey + ":" + relativePathEmpty)).thenReturn(sonarFile);
+  }
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java
index 068c1af82e0..f3a1d51c990 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java
@@ -51,6 +51,8 @@ public class FileMetadataTest {
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(0);
     assertThat(metadata.hash).isNotEmpty();
+    assertThat(metadata.originalLineOffsets).containsOnly(0);
+    assertThat(metadata.lineChecksum).isEmpty();
   }
 
   @Test
@@ -61,6 +63,8 @@ public class FileMetadataTest {
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(3);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10);
+    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 193487042);
   }
 
   @Test
@@ -71,6 +75,8 @@ public class FileMetadataTest {
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(NON_ASCII);
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18);
+    assertThat(metadata.lineChecksum).containsOnly(2090410746, 2090243139, -931663839, 5381);
   }
 
   @Test
@@ -81,6 +87,8 @@ public class FileMetadataTest {
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_16);
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(NON_ASCII);
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18);
+    assertThat(metadata.lineChecksum).containsOnly(2090410746, 2090243139, -931663839, 5381);
   }
 
   @Test
@@ -91,6 +99,8 @@ public class FileMetadataTest {
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(3);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8);
+    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 193487042);
   }
 
   @Test
@@ -101,6 +111,8 @@ public class FileMetadataTest {
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8, 12);
+    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 2090105100, 5381);
   }
 
   @Test
@@ -111,6 +123,8 @@ public class FileMetadataTest {
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9, 13);
+    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 2090105100, 5381);
   }
 
   @Test
@@ -121,6 +135,8 @@ public class FileMetadataTest {
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(3);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9);
+    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 193487042);
   }
 
   @Test
@@ -131,6 +147,20 @@ public class FileMetadataTest {
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_NEW_LINE_FIRST);
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 1, 5, 10);
+    assertThat(metadata.lineChecksum).containsOnly(177583, 2090263731, 2090104836, 193487042);
+  }
+
+  @Test
+  public void start_with_bom() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "\uFEFFfoo\nbar\r\nbaz", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9);
+    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 193487042);
   }
 
   @Test
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml
new file mode 100644
index 00000000000..972ba28b50f
--- /dev/null
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml
@@ -0,0 +1,5 @@
+<dataset>
+
+  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="2014-10-10 16:44:02.000" updated_at="2014-10-10 16:44:02.000" />
+
+</dataset>
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/shared.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/shouldSaveSource.xml
index 7ce1fb033d9..b43de7b75d3 100644
--- a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/shared.xml
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/shouldSaveSource.xml
@@ -7,5 +7,5 @@
   <snapshots purge_status="[null]" period1_mode="[null]" period1_param="[null]" period1_date="[null]" period2_mode="[null]" period2_param="[null]" period2_date="[null]" period3_mode="[null]" period3_param="[null]" period3_date="[null]" period4_mode="[null]" period4_param="[null]" period4_date="[null]" period5_mode="[null]" period5_param="[null]" period5_date="[null]" id="1000" project_id="200" parent_snapshot_id="[null]" root_project_id="100" root_snapshot_id="[null]"
              scope="FIL" qualifier="CLA" created_at="2008-11-01 13:58:00.00" build_date="2008-11-01 13:58:00.00" version="[null]" path=""
              status="U" islast="[false]" depth="3" />
-
+             
 </dataset>
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml
new file mode 100644
index 00000000000..972ba28b50f
--- /dev/null
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml
@@ -0,0 +1,5 @@
+<dataset>
+
+  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="2014-10-10 16:44:02.000" updated_at="2014-10-10 16:44:02.000" />
+
+</dataset>
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml
new file mode 100644
index 00000000000..c6b256b4e44
--- /dev/null
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml
@@ -0,0 +1,7 @@
+<dataset>
+    <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="2014-10-10 16:44:02.000" updated_at="2014-10-10 16:44:02.000" />
+    <file_sources id="102" project_uuid="projectUuid" file_uuid="uuidempty" data="" data_hash="d41d8cd98f00b204e9800998ecf8427e" created_at="2014-10-29 16:44:02.000" updated_at="2014-10-29 16:44:02.000" />
+</dataset>
+
+
+
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistNewFileNoScmNoHighlighting-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistNewFileNoScmNoHighlighting-result.xml
new file mode 100644
index 00000000000..782ada21c80
--- /dev/null
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistNewFileNoScmNoHighlighting-result.xml
@@ -0,0 +1,7 @@
+<dataset>
+    <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="2014-10-10 16:44:02.000" updated_at="2014-10-10 16:44:02.000" />
+    <file_sources id="102" project_uuid="projectUuid" file_uuid="uuidnew" data=",,,,foo&#13;&#10;,,,,bar&#13;&#10;,,,,biz&#13;&#10;" data_hash="0c43ed6418d690ee0ffc3e43e6660967" created_at="2014-10-29 16:44:02.000" updated_at="2014-10-29 16:44:02.000" />
+</dataset>
+
+
+
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistNewFileWithScmAndHighlighting-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistNewFileWithScmAndHighlighting-result.xml
new file mode 100644
index 00000000000..6f3788dfabd
--- /dev/null
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistNewFileWithScmAndHighlighting-result.xml
@@ -0,0 +1,7 @@
+<dataset>
+    <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="2014-10-10 16:44:02.000" updated_at="2014-10-10 16:44:02.000" />
+    <file_sources id="102" project_uuid="projectUuid" file_uuid="uuidnew" data="123,julien,2014-10-11T16:44:02+0100,&#34;0,3,a&#34;,foo&#13;&#10;234,simon,2014-10-12T16:44:02+0100,&#34;0,1,cd&#34;,bar&#13;&#10;345,julien,2014-10-13T16:44:02+0100,&#34;0,9,c&#34;,biz&#13;&#10;" data_hash="a2aaee165e33957a67331fb9f869e0f1" created_at="2014-10-29 16:44:02.000" updated_at="2014-10-29 16:44:02.000" />
+</dataset>
+
+
+
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistUpdateChanged-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistUpdateChanged-result.xml
new file mode 100644
index 00000000000..dfc771b0722
--- /dev/null
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistUpdateChanged-result.xml
@@ -0,0 +1,5 @@
+<dataset>
+
+  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,changed&#13;&#10;,,,,content&#13;&#10;" data_hash="e41cca9c51ff853c748f708f39dfc035" created_at="2014-10-10 16:44:02.000" updated_at="2014-10-29 16:44:02.000" />
+
+</dataset>