SONAR-5868 Allow issue tracking mechanism to work without full previous sources

6 files changed, 67 insertions, 21 deletions

diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
index 8cb3ec22724..7452dc2995e 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
@@ -130,7 +130,9 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     String relativePathSame = "src/changed.java";
     java.io.File sameFile = new java.io.File(basedir, relativePathSame);
     FileUtils.write(sameFile, "changed\ncontent");
-    DefaultInputFile inputFileNew = new DefaultInputFile(PROJECT_KEY, relativePathSame).setLines(2).setAbsolutePath(sameFile.getAbsolutePath());
+    DefaultInputFile inputFileNew = new DefaultInputFile(PROJECT_KEY, relativePathSame).setLines(2)
+      .setAbsolutePath(sameFile.getAbsolutePath())
+      .setLineHashes(new String[] {"foo", "bar"});
     when(inputPathCache.all()).thenReturn(Arrays.<InputPath>asList(inputFileNew));
 
     mockResourceCache(relativePathSame, PROJECT_KEY, "uuidsame");
@@ -142,6 +144,7 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getData()).isEqualTo(
       ",,,,changed\r\n,,,,content\r\n");
+    assertThat(fileSourceDto.getLineHashes()).isEqualTo("foo\nbar");
     assertThat(fileSourceDto.getDataHash()).isEqualTo("e41cca9c51ff853c748f708f39dfc035");
   }
 
@@ -151,7 +154,9 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     when(system2.newDate()).thenReturn(DateUtils.parseDateTime("2014-10-29T16:44:02+0100"));
 
     String relativePathEmpty = "src/empty.java";
-    DefaultInputFile inputFileEmpty = new DefaultInputFile(PROJECT_KEY, relativePathEmpty).setLines(0);
+    DefaultInputFile inputFileEmpty = new DefaultInputFile(PROJECT_KEY, relativePathEmpty)
+      .setLines(0)
+      .setLineHashes(new String[] {});
     when(inputPathCache.all()).thenReturn(Arrays.<InputPath>asList(inputFileEmpty));
 
     mockResourceCache(relativePathEmpty, PROJECT_KEY, "uuidempty");
@@ -169,7 +174,10 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     String relativePathNew = "src/new.java";
     java.io.File newFile = new java.io.File(basedir, relativePathNew);
     FileUtils.write(newFile, "foo\nbar\nbiz");
-    DefaultInputFile inputFileNew = new DefaultInputFile(PROJECT_KEY, relativePathNew).setLines(3).setAbsolutePath(newFile.getAbsolutePath());
+    DefaultInputFile inputFileNew = new DefaultInputFile(PROJECT_KEY, relativePathNew)
+      .setLines(3)
+      .setAbsolutePath(newFile.getAbsolutePath())
+      .setLineHashes(new String[] {"foo", "bar", "bee"});
     when(inputPathCache.all()).thenReturn(Arrays.<InputPath>asList(inputFileNew));
 
     mockResourceCache(relativePathNew, PROJECT_KEY, "uuidnew");
@@ -180,6 +188,7 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getData()).isEqualTo(
       ",,,,foo\r\n,,,,bar\r\n,,,,biz\r\n");
+    assertThat(fileSourceDto.getLineHashes()).isEqualTo("foo\nbar\nbee");
     assertThat(fileSourceDto.getDataHash()).isEqualTo("0c43ed6418d690ee0ffc3e43e6660967");
 
   }
@@ -196,7 +205,8 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     DefaultInputFile inputFileNew = new DefaultInputFile(PROJECT_KEY, relativePathNew)
       .setLines(3)
       .setAbsolutePath(newFile.getAbsolutePath())
-      .setOriginalLineOffsets(new long[] {0, 4, 7});
+      .setOriginalLineOffsets(new long[] {0, 4, 7})
+      .setLineHashes(new String[] {"foo", "bar", "bee"});
     when(inputPathCache.all()).thenReturn(Arrays.<InputPath>asList(inputFileNew));
 
     mockResourceCache(relativePathNew, PROJECT_KEY, "uuidnew");
@@ -221,6 +231,7 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     FileSourceDto fileSourceDto = new FileSourceDao(getMyBatis()).select("uuidnew");
     assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime());
+    assertThat(fileSourceDto.getLineHashes()).isEqualTo("foo\nbar\nbee");
     assertThat(fileSourceDto.getData()).isEqualTo(
       "123,julien,2014-10-11T16:44:02+0100,\"0,3,a\",foo\r\n"
         + "234,simon,2014-10-12T16:44:02+0100,\"0,1,cd\",bar\r\n"
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java
index f3a1d51c990..6814f6888ed 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java
@@ -20,6 +20,7 @@
 package org.sonar.batch.scan.filesystem;
 
 import com.google.common.base.Charsets;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.junit.Rule;
 import org.junit.Test;
@@ -52,7 +53,7 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(0);
     assertThat(metadata.hash).isNotEmpty();
     assertThat(metadata.originalLineOffsets).containsOnly(0);
-    assertThat(metadata.lineChecksum).isEmpty();
+    assertThat(metadata.lineHashes).isEmpty();
   }
 
   @Test
@@ -64,7 +65,7 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(3);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
     assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10);
-    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 193487042);
+    assertThat(metadata.lineHashes).containsOnly(md5("foo"), md5("bar"), md5("baz"));
   }
 
   @Test
@@ -76,7 +77,7 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(NON_ASCII);
     assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18);
-    assertThat(metadata.lineChecksum).containsOnly(2090410746, 2090243139, -931663839, 5381);
+    assertThat(metadata.lineHashes).containsOnly(md5("föo"), md5("bàr"), md5("\u1D11Ebaßz"), "");
   }
 
   @Test
@@ -88,7 +89,7 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(NON_ASCII);
     assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18);
-    assertThat(metadata.lineChecksum).containsOnly(2090410746, 2090243139, -931663839, 5381);
+    assertThat(metadata.lineHashes).containsOnly(md5("föo"), md5("bàr"), md5("\u1D11Ebaßz"), "");
   }
 
   @Test
@@ -100,7 +101,7 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(3);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8);
-    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 193487042);
+    assertThat(metadata.lineHashes).containsOnly(md5("foo"), md5("bar"), md5("baz"));
   }
 
   @Test
@@ -112,7 +113,7 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8, 12);
-    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 2090105100, 5381);
+    assertThat(metadata.lineHashes).containsOnly(md5("foo"), md5("bar"), md5("baz"), "");
   }
 
   @Test
@@ -124,7 +125,7 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITH_LATEST_EOL);
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9, 13);
-    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 2090105100, 5381);
+    assertThat(metadata.lineHashes).containsOnly(md5("foo"), md5("bar"), md5("baz"), "");
   }
 
   @Test
@@ -136,7 +137,7 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(3);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9);
-    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 193487042);
+    assertThat(metadata.lineHashes).containsOnly(md5("foo"), md5("bar"), md5("baz"));
   }
 
   @Test
@@ -148,7 +149,7 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(4);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_NEW_LINE_FIRST);
     assertThat(metadata.originalLineOffsets).containsOnly(0, 1, 5, 10);
-    assertThat(metadata.lineChecksum).containsOnly(177583, 2090263731, 2090104836, 193487042);
+    assertThat(metadata.lineHashes).containsOnly("", md5("foo"), md5("bar"), md5("baz"));
   }
 
   @Test
@@ -160,7 +161,17 @@ public class FileMetadataTest {
     assertThat(metadata.lines).isEqualTo(3);
     assertThat(metadata.hash).isEqualTo(EXPECTED_HASH_WITHOUT_LATEST_EOL);
     assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9);
-    assertThat(metadata.lineChecksum).containsOnly(2090263731, 2090104836, 193487042);
+    assertThat(metadata.lineHashes).containsOnly(md5("foo"), md5("bar"), md5("baz"));
+  }
+
+  @Test
+  public void ignore_whitespace_when_computing_line_hashes() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, " foo\nb ar\r\nbaz \t", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = FileMetadata.INSTANCE.read(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3);
+    assertThat(metadata.lineHashes).containsOnly(md5("foo"), md5("bar"), md5("baz"));
   }
 
   @Test
@@ -192,4 +203,8 @@ public class FileMetadataTest {
     assertThat(hash1).isEqualTo(hash1a);
     assertThat(hash1).isNotEqualTo(hash2);
   }
+
+  private static String md5(String input) {
+    return DigestUtils.md5Hex(input);
+  }
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputPathCacheTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputPathCacheTest.java
index 7008b0450c0..bbb63290841 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputPathCacheTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputPathCacheTest.java
@@ -67,10 +67,16 @@ public class InputPathCacheTest {
       .setStatus(Status.ADDED)
       .setHash("xyz")
       .setLines(1)
+      .setEncoding("UTF-8")
+      .setOriginalLineOffsets(new long[] {0, 4})
+      .setLineHashes(new String[] {"foo", "bar"})
       .setFile(temp.newFile("Bar.java")));
 
-    assertThat(cache.getFile("struts", "src/main/java/Foo.java").relativePath())
-      .isEqualTo("src/main/java/Foo.java");
+    DefaultInputFile loadedFile = (DefaultInputFile) cache.getFile("struts-core", "src/main/java/Bar.java");
+    assertThat(loadedFile.relativePath()).isEqualTo("src/main/java/Bar.java");
+    assertThat(loadedFile.encoding()).isEqualTo("UTF-8");
+    assertThat(loadedFile.originalLineOffsets()).containsOnly(0, 4);
+    assertThat(loadedFile.lineHashes()).containsOnly("foo", "bar");
 
     assertThat(cache.filesByModule("struts")).hasSize(1);
     assertThat(cache.filesByModule("struts-core")).hasSize(1);
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml
index fae5b462879..a4e8ed467b3 100644
--- a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml
@@ -1,5 +1,8 @@
 <dataset>
-
-  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="1412952242000" updated_at="1412952242000" />
+  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" 
+      data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" 
+      line_hashes="8d7b3d6b83c0a517eac07e1aac94b773&#10;9a0364b9e99bb480dd25e1f0284c8555" 
+      data_hash="ee716d4ed9faae16eb9167714442a3bc" 
+      created_at="1412952242000" updated_at="1412952242000" />
 
 </dataset>
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml
index fae5b462879..98dd81f73df 100644
--- a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml
@@ -1,5 +1,9 @@
 <dataset>
 
-  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="1412952242000" updated_at="1412952242000" />
+  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" 
+      data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" 
+      line_hashes="8d7b3d6b83c0a517eac07e1aac94b773&#10;9a0364b9e99bb480dd25e1f0284c8555" 
+      data_hash="ee716d4ed9faae16eb9167714442a3bc" 
+      created_at="1412952242000" updated_at="1412952242000" />
 
 </dataset>
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml
index ac3c16563af..cd844a149d8 100644
--- a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml
+++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml
@@ -1,6 +1,13 @@
 <dataset>
-    <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="1412952242000" updated_at="1412952242000" />
-    <file_sources id="102" project_uuid="projectUuid" file_uuid="uuidempty" data="[null]" data_hash="0" created_at="1414597442000" updated_at="1414597442000" />
+    <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" 
+      data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" 
+      line_hashes="8d7b3d6b83c0a517eac07e1aac94b773&#10;9a0364b9e99bb480dd25e1f0284c8555" 
+      data_hash="ee716d4ed9faae16eb9167714442a3bc" 
+      created_at="1412952242000" updated_at="1412952242000" />
+      
+    <file_sources id="102" project_uuid="projectUuid" file_uuid="uuidempty" data="[null]"
+       line_hashes="[null]" 
+       data_hash="0" created_at="1414597442000" updated_at="1414597442000" />
 </dataset>