SONAR-5815 Store last update time on source data

sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/input/ProjectReferentials.java

@@ -20,12 +20,15 @@
 package org.sonar.batch.protocol.input;
 
 import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 
 import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
 
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -40,6 +43,7 @@ public class ProjectReferentials {
   private Collection<ActiveRule> activeRules = new ArrayList<ActiveRule>();
   private Map<String, Map<String, String>> settingsByModule = new HashMap<String, Map<String, String>>();
   private Map<String, Map<String, FileData>> fileDataByModuleAndPath = new HashMap<String, Map<String, FileData>>();
+  private Date lastAnalysisDate;
 
   public Map<String, String> settings(String projectKey) {
     return settingsByModule.containsKey(projectKey) ? settingsByModule.get(projectKey) : Collections.<String, String>emptyMap();
@@ -100,12 +104,23 @@ public class ProjectReferentials {
     this.timestamp = timestamp;
   }
 
+  @CheckForNull
+  public Date lastAnalysisDate() {
+    return lastAnalysisDate;
+  }
+
+  public void setLastAnalysisDate(@Nullable Date lastAnalysisDate) {
+    this.lastAnalysisDate = lastAnalysisDate;
+  }
+
   public String toJson() {
-    return new Gson().toJson(this);
+    Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create();
+    return gson.toJson(this);
   }
 
   public static ProjectReferentials fromJson(String json) {
-    return new Gson().fromJson(json, ProjectReferentials.class);
+    Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create();
+    return gson.fromJson(json, ProjectReferentials.class);
   }
 
 }

sonar-batch-protocol/src/test/java/org/sonar/batch/protocol/input/ProjectReferentialsTest.java

@@ -48,6 +48,7 @@ public class ProjectReferentialsTest {
     ActiveRule activeRule = new ActiveRule("repo", "rule", "Rule", "MAJOR", "rule", "java");
     activeRule.addParam("param1", "value1");
     ref.addActiveRule(activeRule);
+    ref.setLastAnalysisDate(new SimpleDateFormat("dd/MM/yyyy").parse("31/10/2014"));
     ref.setTimestamp(10);
     ref.addFileData("foo", "src/main/java/Foo.java", new FileData("xyz", "1=12345,2=3456", "1=345,2=345", "1=henryju,2=gaudin"));
 
@@ -55,10 +56,11 @@ public class ProjectReferentialsTest {
     JSONAssert
       .assertEquals(
         "{timestamp:10,"
-          + "qprofilesByLanguage:{java:{key:\"squid-java\",name:Java,language:java,rulesUpdatedAt:\"Mar 14, 1984 12:00:00 AM\"}},"
+          + "qprofilesByLanguage:{java:{key:\"squid-java\",name:Java,language:java,rulesUpdatedAt:\"1984-03-14T00:00:00+0100\"}},"
           + "activeRules:[{repositoryKey:repo,ruleKey:rule,name:Rule,severity:MAJOR,internalKey:rule,language:java,params:{param1:value1}}],"
           + "settingsByModule:{foo:{prop1:value1,prop2:value2,prop:value}},"
-          + "fileDataByModuleAndPath:{foo:{\"src/main/java/Foo.java\":{hash:xyz,scmLastCommitDatetimesByLine:\"1\u003d12345,2\u003d3456\",scmRevisionsByLine:\"1\u003d345,2\u003d345\",scmAuthorsByLine:\"1\u003dhenryju,2\u003dgaudin\"}}}}",
+          + "fileDataByModuleAndPath:{foo:{\"src/main/java/Foo.java\":{hash:xyz,scmLastCommitDatetimesByLine:\"1\u003d12345,2\u003d3456\",scmRevisionsByLine:\"1\u003d345,2\u003d345\",scmAuthorsByLine:\"1\u003dhenryju,2\u003dgaudin\"}}},"
+          + "lastAnalysisDate:\"2014-10-31T00:00:00+0100\"}",
         ref.toJson(), true);
   }
 
@@ -66,10 +68,11 @@ public class ProjectReferentialsTest {
   public void testFromJson() throws JSONException, ParseException {
     ProjectReferentials ref = ProjectReferentials
       .fromJson("{timestamp:1,"
-        + "qprofilesByLanguage:{java:{key:\"squid-java\",name:Java,language:java,rulesUpdatedAt:\"Mar 14, 1984 12:00:00 AM\"}},"
+        + "qprofilesByLanguage:{java:{key:\"squid-java\",name:Java,language:java,rulesUpdatedAt:\"1984-03-14T00:00:00+0100\"}},"
         + "activeRules:[{repositoryKey:repo,ruleKey:rule,name:Rule,severity:MAJOR,internalKey:rule1,language:java,params:{param1:value1}}],"
         + "settingsByModule:{foo:{prop:value}},"
-        + "fileDataByModuleAndPath:{foo:{\"src/main/java/Foo.java\":{hash:xyz,scmLastCommitDatetimesByLine:\"1\u003d12345,2\u003d3456\",scmRevisionsByLine:\"1\u003d345,2\u003d345\",scmAuthorsByLine:\"1\u003dhenryju,2\u003dgaudin\"}}}}");
+        + "fileDataByModuleAndPath:{foo:{\"src/main/java/Foo.java\":{hash:xyz,scmLastCommitDatetimesByLine:\"1\u003d12345,2\u003d3456\",scmRevisionsByLine:\"1\u003d345,2\u003d345\",scmAuthorsByLine:\"1\u003dhenryju,2\u003dgaudin\"}}},"
+        + "lastAnalysisDate:\"2014-10-31T00:00:00+0100\"}");
 
     assertThat(ref.timestamp()).isEqualTo(1);
 
@@ -92,5 +95,7 @@ public class ProjectReferentialsTest {
     assertThat(ref.fileData("foo", "src/main/java/Foo.java").scmAuthorsByLine()).isEqualTo("1=henryju,2=gaudin");
     assertThat(ref.fileData("foo", "src/main/java/Foo.java").scmLastCommitDatetimesByLine()).isEqualTo("1=12345,2=3456");
     assertThat(ref.fileData("foo", "src/main/java/Foo.java").scmRevisionsByLine()).isEqualTo("1=345,2=345");
+
+    assertThat(ref.lastAnalysisDate()).isEqualTo(new SimpleDateFormat("dd/MM/yyyy").parse("31/10/2014"));
   }
 }

sonar-batch/.sonar/profiling/myProject-profiler.xml

@@ -10,9 +10,9 @@
 <entry key="Post-Jobs">30</entry>
 <entry key="FakeSensor">10</entry>
 <entry key="Maven">4</entry>
-<entry key="Free memory">9</entry>
-<entry key="Persisters">40</entry>
 <entry key="Decorators">30</entry>
-<entry key="FakeInitializer">7</entry>
+<entry key="Persisters">40</entry>
+<entry key="Free memory">9</entry>
 <entry key="Sensors">10</entry>
+<entry key="FakeInitializer">7</entry>
 </properties>

sonar-batch/.sonar/profiling/total-execution-profiler.xml

@@ -10,9 +10,9 @@
 <entry key="Post-Jobs">90</entry>
 <entry key="FakeSensor">30</entry>
 <entry key="Maven">12</entry>
-<entry key="Free memory">27</entry>
-<entry key="Persisters">120</entry>
 <entry key="Decorators">90</entry>
-<entry key="FakeInitializer">21</entry>
+<entry key="Persisters">120</entry>
+<entry key="Free memory">27</entry>
 <entry key="Sensors">30</entry>
+<entry key="FakeInitializer">21</entry>
 </properties>

sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java

@@ -474,14 +474,6 @@ public class DefaultIndex extends SonarIndex {
     return null;
   }
 
-  @Override
-  public void setSource(Resource reference, String source) {
-    Bucket bucket = getBucket(reference);
-    if (bucket != null) {
-      persistence.setSource(reference, source);
-    }
-  }
-
   @Override
   public String getSource(Resource resource) {
     return persistence.getSource(resource);

sonar-batch/src/main/java/org/sonar/batch/index/DefaultPersistenceManager.java

@@ -51,7 +51,6 @@ public final class DefaultPersistenceManager implements PersistenceManager {
   @Override
   public void clear() {
     resourcePersister.clear();
-    sourcePersister.clear();
   }
 
   @Override
@@ -67,11 +66,6 @@ public final class DefaultPersistenceManager implements PersistenceManager {
     return null;
   }
 
-  @Override
-  public void setSource(Resource file, String source) {
-    sourcePersister.saveSource(file, source);
-  }
-
   @Override
   public String getSource(Resource resource) {
     return sourcePersister.getSource(resource);

sonar-batch/src/main/java/org/sonar/batch/index/PersistenceManager.java

@@ -37,8 +37,6 @@ public interface PersistenceManager {
 
   Snapshot saveResource(Project project, Resource resource, @Nullable Resource parent);
 
-  void setSource(Resource file, String source);
-
   String getSource(Resource resource);
 
   void saveDependency(Project project, Dependency dependency, Dependency parentDependency);

sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java

@@ -19,20 +19,17 @@
  */
 package org.sonar.batch.index;
 
-import com.google.common.collect.Sets;
 import org.sonar.api.database.model.Snapshot;
-import org.sonar.api.resources.DuplicatedSourceException;
 import org.sonar.api.resources.Resource;
 import org.sonar.core.source.db.SnapshotSourceDao;
 import org.sonar.core.source.db.SnapshotSourceDto;
 
 import javax.annotation.CheckForNull;
 
-import java.util.Set;
+import java.util.Date;
 
-public final class SourcePersister {
+public class SourcePersister {
 
-  private Set<Integer> savedSnapshotIds = Sets.newHashSet();
   private ResourcePersister resourcePersister;
   private final SnapshotSourceDao sourceDao;
 
@@ -41,16 +38,13 @@ public final class SourcePersister {
     this.sourceDao = sourceDao;
   }
 
-  public void saveSource(Resource resource, String source) {
+  public void saveSource(Resource resource, String source, Date updatedAt) {
     Snapshot snapshot = resourcePersister.getSnapshotOrFail(resource);
-    if (isCached(snapshot)) {
-      throw new DuplicatedSourceException(resource);
-    }
     SnapshotSourceDto dto = new SnapshotSourceDto();
     dto.setSnapshotId(snapshot.getId().longValue());
     dto.setData(source);
+    dto.setUpdatedAt(updatedAt);
     sourceDao.insert(dto);
-    addToCache(snapshot);
   }
 
   @CheckForNull
@@ -61,16 +55,4 @@ public final class SourcePersister {
     }
     return null;
   }
-
-  private boolean isCached(Snapshot snapshot) {
-    return savedSnapshotIds.contains(snapshot.getId());
-  }
-
-  private void addToCache(Snapshot snapshot) {
-    savedSnapshotIds.add(snapshot.getId());
-  }
-
-  public void clear() {
-    savedSnapshotIds.clear();
-  }
 }

sonar-batch/src/main/java/org/sonar/batch/referential/DefaultProjectReferentialsLoader.java

@@ -42,10 +42,13 @@ import org.sonar.core.source.SnapshotDataTypes;
 import org.sonar.core.source.db.SnapshotDataDao;
 import org.sonar.core.source.db.SnapshotDataDto;
 
+import javax.annotation.CheckForNull;
+import javax.persistence.NoResultException;
 import javax.persistence.Query;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 
@@ -102,6 +105,7 @@ public class DefaultProjectReferentialsLoader implements ProjectReferentialsLoad
         ref.addFileData(module.getKeyWithBranch(), path, new FileData(hash, lastCommits, revisions, authors));
       }
     }
+    ref.setLastAnalysisDate(lastSnapshotCreationDate(projectKey));
     return ref;
   }
 
@@ -154,4 +158,33 @@ public class DefaultProjectReferentialsLoader implements ProjectReferentialsLoad
     }
     return jpaQuery.getResultList();
   }
+
+  @CheckForNull
+  Date lastSnapshotCreationDate(String resourceKey) {
+    StringBuilder sb = new StringBuilder();
+    Map<String, Object> params = Maps.newHashMap();
+
+    sb.append("SELECT s.buildDate");
+    sb.append(" FROM ")
+      .append(ResourceModel.class.getSimpleName())
+      .append(" r, ")
+      .append(Snapshot.class.getSimpleName())
+      .append(" s WHERE s.resourceId=r.id AND r.key=:kee AND s.status=:status AND s.qualifier<>:lib");
+    params.put("kee", resourceKey);
+    params.put("status", Snapshot.STATUS_PROCESSED);
+    params.put("lib", Qualifiers.LIBRARY);
+
+    sb.append(" AND s.last=true ");
+
+    Query jpaQuery = session.createQuery(sb.toString());
+
+    for (Map.Entry<String, Object> entry : params.entrySet()) {
+      jpaQuery.setParameter(entry.getKey(), entry.getValue());
+    }
+    try {
+      return (Date) jpaQuery.getSingleResult();
+    } catch (NoResultException e) {
+      return null;
+    }
+  }
 }

sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java

@@ -26,6 +26,7 @@ import org.sonar.api.BatchComponent;
 import org.sonar.api.batch.SonarIndex;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.InputFile.Status;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
 import org.sonar.api.resources.File;
 import org.sonar.api.resources.Languages;
@@ -33,8 +34,13 @@ import org.sonar.api.resources.Project;
 import org.sonar.api.resources.Resource;
 import org.sonar.api.utils.SonarException;
 import org.sonar.batch.index.ResourceKeyMigration;
+import org.sonar.batch.index.SnapshotCache;
+import org.sonar.batch.index.SourcePersister;
+import org.sonar.batch.protocol.input.ProjectReferentials;
 import org.sonar.batch.util.DeprecatedKeyUtils;
 
+import java.util.Date;
+
 /**
  * Index all files/directories of the module in SQ database and importing source code.
  *
@@ -46,12 +52,19 @@ public class ComponentIndexer implements BatchComponent {
   private final SonarIndex sonarIndex;
   private final ResourceKeyMigration migration;
   private final Project module;
+  private final SourcePersister sourcePersister;
+  private final ProjectReferentials projectReferentials;
+  private final Date projectAnalysisDate;
 
-  public ComponentIndexer(Project module, Languages languages, SonarIndex sonarIndex, ResourceKeyMigration migration) {
+  public ComponentIndexer(Project module, Languages languages, SonarIndex sonarIndex, ResourceKeyMigration migration, SourcePersister sourcePersister,
+    ProjectReferentials projectReferentials, SnapshotCache snapshotCache) {
     this.module = module;
     this.languages = languages;
     this.sonarIndex = sonarIndex;
     this.migration = migration;
+    this.sourcePersister = sourcePersister;
+    this.projectReferentials = projectReferentials;
+    this.projectAnalysisDate = snapshotCache.get(module.getEffectiveKey()).getBuildDate();
   }
 
   public void execute(FileSystem fs) {
@@ -80,12 +93,16 @@ public class ComponentIndexer implements BatchComponent {
   void importSources(FileSystem fs, InputFile inputFile, Resource sonarFile) {
     try {
       // TODO this part deserves optimization.
-      // No need to read full content in memory when shouldImportSource=false
       // We should try to remove BOM and count lines in a single pass
       String source = Files.toString(inputFile.file(), fs.encoding());
       // SONAR-3860 Remove BOM character from source
       source = CharMatcher.anyOf("\uFEFF").removeFrom(source);
-      sonarIndex.setSource(sonarFile, source);
+      if (inputFile.status() == Status.SAME) {
+        sourcePersister.saveSource(sonarFile, source, projectReferentials.lastAnalysisDate());
+      } else {
+        sourcePersister.saveSource(sonarFile, source, this.projectAnalysisDate);
+      }
+
     } catch (Exception e) {
       throw new SonarException("Unable to read and import the source file : '" + inputFile.absolutePath() + "' with the charset : '"
         + fs.encoding() + "'.", e);

sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java

@@ -22,9 +22,9 @@ package org.sonar.batch.index;
 import org.junit.Before;
 import org.junit.Test;
 import org.sonar.api.database.model.Snapshot;
-import org.sonar.api.resources.DuplicatedSourceException;
 import org.sonar.api.resources.File;
 import org.sonar.api.resources.Resource;
+import org.sonar.api.utils.DateUtils;
 import org.sonar.core.persistence.AbstractDaoTestCase;
 import org.sonar.core.source.db.SnapshotSourceDao;
 
@@ -48,14 +48,8 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
 
   @Test
   public void shouldSaveSource() {
-    sourcePersister.saveSource(new File("org/foo/Bar.java"), "this is the file content");
+    sourcePersister.saveSource(new File("org/foo/Bar.java"), "this is the file content", DateUtils.parseDateTime("2014-10-31T16:44:02+0100"));
     checkTables("shouldSaveSource", "snapshot_sources");
   }
 
-  @Test(expected = DuplicatedSourceException.class)
-  public void shouldFailIfSourceSavedSeveralTimes() {
-    File file = new File("org/foo/Bar.java");
-    sourcePersister.saveSource(file, "this is the file content");
-    sourcePersister.saveSource(file, "new content"); // fail
-  }
 }

sonar-batch/src/test/java/org/sonar/batch/referential/DefaultProjectReferentialsLoaderTest.java

@@ -32,7 +32,9 @@ import org.sonar.batch.rule.ModuleQProfiles;
 import org.sonar.core.source.db.SnapshotDataDao;
 
 import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -49,7 +51,9 @@ public class DefaultProjectReferentialsLoaderTest {
     serverClient = mock(ServerClient.class);
     analysisMode = mock(AnalysisMode.class);
     loader = new DefaultProjectReferentialsLoader(mock(DatabaseSession.class), serverClient, analysisMode, mock(SnapshotDataDao.class));
-    when(serverClient.request(anyString())).thenReturn("");
+    loader = spy(loader);
+    doReturn(null).when(loader).lastSnapshotCreationDate(anyString());
+    when(serverClient.request(anyString())).thenReturn("{}");
     reactor = new ProjectReactor(ProjectDefinition.create().setKey("foo"));
     taskProperties = new TaskProperties(Maps.<String, String>newHashMap(), "");
   }

sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ComponentIndexerTest.java

@@ -33,6 +33,7 @@ import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
+import org.sonar.api.database.model.Snapshot;
 import org.sonar.api.resources.AbstractLanguage;
 import org.sonar.api.resources.Java;
 import org.sonar.api.resources.Languages;
@@ -40,10 +41,14 @@ import org.sonar.api.resources.Project;
 import org.sonar.api.resources.Qualifiers;
 import org.sonar.api.resources.Resource;
 import org.sonar.batch.index.ResourceKeyMigration;
+import org.sonar.batch.index.SnapshotCache;
+import org.sonar.batch.index.SourcePersister;
+import org.sonar.batch.protocol.input.ProjectReferentials;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
+import java.util.Date;
 
 import static org.mockito.Matchers.argThat;
 import static org.mockito.Matchers.eq;
@@ -62,12 +67,14 @@ public class ComponentIndexerTest {
 
   String aClaess;
   String explicacao;
+  private SourcePersister sourcePersister;
 
   @Before
   public void prepare() throws IOException {
     baseDir = temp.newFolder();
     sonarIndex = mock(SonarIndex.class);
-    project = mock(Project.class);
+    sourcePersister = mock(SourcePersister.class);
+    project = new Project("myProject");
     cobolLanguage = new AbstractLanguage("cobol") {
       @Override
       public String[] getFileSuffixes() {
@@ -85,7 +92,7 @@ public class ComponentIndexerTest {
     fs.add(newInputFile("src/main/java2/foo/bar/Foo.java", "", "foo/bar/Foo.java", "java", false));
     fs.add(newInputFile("src/test/java/foo/bar/FooTest.java", "", "foo/bar/FooTest.java", "java", true));
     Languages languages = new Languages(Java.INSTANCE);
-    ComponentIndexer indexer = new ComponentIndexer(project, languages, sonarIndex, mock(ResourceKeyMigration.class));
+    ComponentIndexer indexer = createIndexer(languages);
     indexer.execute(fs);
 
     verify(sonarIndex).index(org.sonar.api.resources.File.create("src/main/java/foo/bar/Foo.java", "foo/bar/Foo.java", Java.INSTANCE, false));
@@ -101,6 +108,14 @@ public class ComponentIndexerTest {
     }));
   }
 
+  private ComponentIndexer createIndexer(Languages languages) {
+    SnapshotCache snapshotCache = new SnapshotCache();
+    snapshotCache.put("myProject", mock(Snapshot.class));
+    ComponentIndexer indexer = new ComponentIndexer(project, languages, sonarIndex, mock(ResourceKeyMigration.class), sourcePersister, new ProjectReferentials(),
+      snapshotCache);
+    return indexer;
+  }
+
   @Test
   public void should_index_cobol_files() throws IOException {
     fs.add(newInputFile("src/foo/bar/Foo.cbl", "", "foo/bar/Foo.cbl", "cobol", false));
@@ -108,7 +123,7 @@ public class ComponentIndexerTest {
     fs.add(newInputFile("src/test/foo/bar/FooTest.cbl", "", "foo/bar/FooTest.cbl", "cobol", true));
 
     Languages languages = new Languages(cobolLanguage);
-    ComponentIndexer indexer = new ComponentIndexer(project, languages, sonarIndex, mock(ResourceKeyMigration.class));
+    ComponentIndexer indexer = createIndexer(languages);
     indexer.execute(fs);
 
     verify(sonarIndex).index(org.sonar.api.resources.File.create("/src/foo/bar/Foo.cbl", "foo/bar/Foo.cbl", cobolLanguage, false));
@@ -120,12 +135,12 @@ public class ComponentIndexerTest {
   public void shouldImportSource() throws IOException {
     fs.add(newInputFile("src/main/java/foo/bar/Foo.java", "sample code", "foo/bar/Foo.java", "java", false));
     Languages languages = new Languages(Java.INSTANCE);
-    ComponentIndexer indexer = new ComponentIndexer(project, languages, sonarIndex, mock(ResourceKeyMigration.class));
+    ComponentIndexer indexer = createIndexer(languages);
     indexer.execute(fs);
 
     Resource sonarFile = org.sonar.api.resources.File.create("src/main/java/foo/bar/Foo.java", "foo/bar/Foo.java", Java.INSTANCE, false);
     verify(sonarIndex).index(sonarFile);
-    verify(sonarIndex).setSource(sonarFile, "sample code");
+    verify(sourcePersister).saveSource(sonarFile, "sample code", null);
   }
 
   @Test
@@ -158,18 +173,18 @@ public class ComponentIndexerTest {
       .setFile(javaFile1)
       .setLanguage("java"));
     Languages languages = new Languages(Java.INSTANCE);
-    ComponentIndexer indexer = new ComponentIndexer(project, languages, sonarIndex, mock(ResourceKeyMigration.class));
+    ComponentIndexer indexer = createIndexer(languages);
     indexer.execute(fs);
 
     Resource sonarFile = org.sonar.api.resources.File.create("src/main/java/foo/bar/Foo.java", "foo/bar/Foo.java", Java.INSTANCE, false);
 
-    verify(sonarIndex).setSource(eq(sonarFile), argThat(new ArgumentMatcher<String>() {
+    verify(sourcePersister).saveSource(eq(sonarFile), argThat(new ArgumentMatcher<String>() {
       @Override
       public boolean matches(Object arg0) {
         String source = (String) arg0;
         return !source.contains("\uFEFF");
       }
-    }));
+    }), (Date) eq(null));
   }
 
   private void fileEncodingTest(String encoding, String testFile) throws Exception {
@@ -182,18 +197,18 @@ public class ComponentIndexerTest {
       .setFile(javaFile1)
       .setLanguage("java"));
     Languages languages = new Languages(Java.INSTANCE);
-    ComponentIndexer indexer = new ComponentIndexer(project, languages, sonarIndex, mock(ResourceKeyMigration.class));
+    ComponentIndexer indexer = createIndexer(languages);
     indexer.execute(fs);
 
     Resource sonarFile = org.sonar.api.resources.File.create("/src/main/java/foo/bar/Foo.java", "foo/bar/Foo.java", Java.INSTANCE, false);
 
-    verify(sonarIndex).setSource(eq(sonarFile), argThat(new ArgumentMatcher<String>() {
+    verify(sourcePersister).saveSource(eq(sonarFile), argThat(new ArgumentMatcher<String>() {
       @Override
       public boolean matches(Object arg0) {
         String source = (String) arg0;
         return source.contains(aClaess) && source.contains(explicacao);
       }
-    }));
+    }), (Date) eq(null));
   }
 
   private File getFile(String testFile) {

sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java

@@ -35,6 +35,7 @@ public class FileMetadataTest {
   private static final String EXPECTED_HASH_WITHOUT_LATEST_EOL = "c80cc50d65ace6c4eb63f189d274dbeb";
   private static final String EXPECTED_HASH_NEW_LINE_FIRST = "cf2d41454b5b451eeb5122f0848c1d2a";
   private static final String EXPECTED_HASH_WITH_LATEST_EOL = "bf77e51d219e7d7d643faac86f1b5d15";
+  private static final String NON_ASCII = "4050369e8ba432c9079e258b43fe4ab5";
 
   @Rule
   public ExpectedException thrown = ExpectedException.none();
@@ -63,13 +64,23 @@ public class FileMetadataTest {
   }
 
   @Test
-  public void windows_with_latest_eol() throws Exception {
+  public void non_ascii_utf_8() throws Exception {
     File tempFile = temp.newFile();
-    FileUtils.write(tempFile, "foo\r\nbar\r\nbaz\r\n", Charsets.UTF_8, true);
+    FileUtils.write(tempFile, "föo\r\nbàr\r\n\u1D11Ebaßz\r\n", Charsets.UTF_8, true);
 
     FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(4);
-    assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
+    assertThat(metadata.hash).isEqualTo(NON_ASCII);
+  }
+
+  @Test
+  public void non_ascii_utf_16() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "föo\r\nbàr\r\n\u1D11Ebaßz\r\n", Charsets.UTF_16, true);
+
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_16);
+    assertThat(metadata.lines).isEqualTo(4);
+    assertThat(metadata.hash).isEqualTo(NON_ASCII);
   }
 
   @Test

sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/shouldSaveSource-result.xml

@@ -8,5 +8,5 @@
              scope="FIL" qualifier="CLA" created_at="2008-11-01 13:58:00.00" build_date="2008-11-01 13:58:00.00" version="[null]" path=""
              status="U" islast="[false]" depth="3" />
 
-  <SNAPSHOT_SOURCES ID="1" SNAPSHOT_ID="1000" DATA="this is the file content" UPDATED_AT="[null]"/>
+  <SNAPSHOT_SOURCES ID="1" SNAPSHOT_ID="1000" DATA="this is the file content" updated_at="2014-10-31 16:44:02.000"/>
 </dataset>

sonar-core/src/main/java/org/sonar/core/source/db/SnapshotSourceDto.java

@@ -19,10 +19,13 @@
  */
 package org.sonar.core.source.db;
 
+import java.util.Date;
+
 public class SnapshotSourceDto {
   private Long id;
   private Long snapshotId;
   private String data;
+  private Date updatedAt;
 
   public Long getId() {
     return id;
@@ -50,4 +53,13 @@ public class SnapshotSourceDto {
     this.data = data;
     return this;
   }
+
+  public Date getUpdatedAt() {
+    return updatedAt;
+  }
+
+  public SnapshotSourceDto setUpdatedAt(Date updatedAt) {
+    this.updatedAt = updatedAt;
+    return this;
+  }
 }

sonar-core/src/main/resources/org/sonar/core/source/db/SnapshotSourceMapper.xml

@@ -19,7 +19,8 @@
   </select>
 
   <insert id="insert" parameterType="org.sonar.core.source.db.SnapshotSourceDto" useGeneratedKeys="false">
-    insert into snapshot_sources (snapshot_id, data) values (#{snapshotId}, #{data})
+    insert into snapshot_sources (snapshot_id, data, updated_at) 
+    values (#{snapshotId}, #{data}, #{updatedAt})
   </insert>
 
 </mapper>

sonar-core/src/test/java/org/sonar/core/source/db/SnapshotSourceDaoTest.java

@@ -22,6 +22,7 @@ package org.sonar.core.source.db;
 
 import org.junit.Before;
 import org.junit.Test;
+import org.sonar.api.utils.DateUtils;
 import org.sonar.core.persistence.AbstractDaoTestCase;
 
 import static org.fest.assertions.Assertions.assertThat;
@@ -52,7 +53,7 @@ public class SnapshotSourceDaoTest extends AbstractDaoTestCase {
 
   @Test
   public void insert() throws Exception {
-    dao.insert(new SnapshotSourceDto().setId(102L).setData("bar").setSnapshotId(11L));
+    dao.insert(new SnapshotSourceDto().setId(102L).setData("bar").setSnapshotId(11L).setUpdatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100")));
 
     checkTable("insert", "snapshot_sources");
   }

sonar-core/src/test/resources/org/sonar/core/source/db/SnapshotSourceDaoTest/insert-result.xml

@@ -5,8 +5,8 @@
   <snapshots id="10" project_id="1" islast="[false]"/>
   <snapshots id="11" project_id="1" islast="[true]"/>
 
-  <snapshot_sources id="101" snapshot_id="11" data="public class Foo {public Foo(){}}" updated_at="[null]"/>
+  <snapshot_sources id="101" snapshot_id="11" data="public class Foo {public Foo(){}}" updated_at="2014-10-30 16:44:02.000"/>
 
-  <snapshot_sources id="102" snapshot_id="11" data="bar" updated_at="[null]"/>
+  <snapshot_sources id="102" snapshot_id="11" data="bar" updated_at="2014-10-31 16:44:02.000"/>
 
 </dataset>

sonar-core/src/test/resources/org/sonar/core/source/db/SnapshotSourceDaoTest/shared.xml

@@ -5,6 +5,6 @@
     <snapshots id="10" project_id="1" islast="[false]" />
     <snapshots id="11" project_id="1" islast="[true]" />
 
-    <snapshot_sources id="101" snapshot_id="11" data="public class Foo {public Foo(){}}" />
+    <snapshot_sources id="101" snapshot_id="11" data="public class Foo {public Foo(){}}" updated_at="2014-10-30 16:44:02.000" />
 
 </dataset>

sonar-plugin-api/src/main/java/org/sonar/api/batch/SonarIndex.java

@@ -93,20 +93,9 @@ public abstract class SonarIndex implements DirectedGraphAccessor<Resource, Depe
 
   public abstract Collection<Resource> getChildren(Resource reference);
 
-  /**
-   * Save the source code of a file. The file must be have been indexed before.
-   * Note: the source stream is not closed.
-   *
-   * @throws org.sonar.api.resources.DuplicatedSourceException
-   *          if the source has already been set on this resource
-   * @deprecated since 4.2 should not be used by plugins
-   */
-  @Deprecated
-  public abstract void setSource(Resource reference, String source);
-
   /**
    * @return source code associated with a specified resource, <code>null</code> if not available 
-   * (for example when sonar.importSources=false)
+   * (for example if resource is not a file)
    * @since 2.9
    */
   @CheckForNull