SONAR-6258 Fix hashes, symbols and highlighting persistence

server/sonar-server/src/main/java/org/sonar/server/computation/source/ComputeFileSourceData.java

@@ -49,23 +49,29 @@ public class ComputeFileSourceData {
   public Data compute() {
     Data data = new Data();
     while (linesIterator.hasNext()) {
-      read(data, linesIterator.next());
+      currentLine++;
+      read(data, linesIterator.next(), hasNextLine());
     }
     // Process last line
-    if (currentLine < numberOfLines) {
-      read(data, "");
+    if (hasNextLine()) {
+      currentLine++;
+      read(data, "", false);
     }
     return data;
   }
 
-  private void read(Data data, String source) {
-    FileSourceDb.Line.Builder lineBuilder = data.fileSourceBuilder.addLinesBuilder().setSource(source);
+  private void read(Data data, String source, boolean hasNextLine) {
+    if (hasNextLine) {
+      data.lineHashes.append(computeLineChecksum(source)).append("\n");
+      data.srcMd5Digest.update((source + "\n").getBytes(UTF_8));
+    } else {
+      data.lineHashes.append(computeLineChecksum(source));
+      data.srcMd5Digest.update(source.getBytes(UTF_8));
+    }
 
-    currentLine++;
-    String sourceLine = lineBuilder.getSource();
-    data.lineHashes.append(computeLineChecksum(sourceLine)).append("\n");
-    data.srcMd5Digest.update((sourceLine + "\n").getBytes(UTF_8));
-    lineBuilder.setLine(currentLine);
+    FileSourceDb.Line.Builder lineBuilder = data.fileSourceBuilder.addLinesBuilder()
+      .setSource(source)
+      .setLine(currentLine);
     for (LineReader lineReader : lineReaders) {
       lineReader.read(lineBuilder);
     }
@@ -79,6 +85,10 @@ public class ComputeFileSourceData {
     return DigestUtils.md5Hex(reducedLine);
   }
 
+  private boolean hasNextLine(){
+    return linesIterator.hasNext() || currentLine < numberOfLines;
+  }
+
   public static class Data {
     private final StringBuilder lineHashes;
     private final MessageDigest srcMd5Digest;

server/sonar-server/src/main/java/org/sonar/server/computation/source/HighlightingLineReader.java

@@ -63,13 +63,22 @@ public class HighlightingLineReader implements LineReader {
     StringBuilder highlighting = new StringBuilder();
 
     incrementHighlightingListMatchingLine(line);
-    for (Iterator<BatchReport.SyntaxHighlighting> syntaxHighlightingIterator = highlightingList.iterator(); syntaxHighlightingIterator.hasNext(); ) {
+    for (Iterator<BatchReport.SyntaxHighlighting> syntaxHighlightingIterator = highlightingList.iterator(); syntaxHighlightingIterator.hasNext();) {
       BatchReport.SyntaxHighlighting syntaxHighlighting = syntaxHighlightingIterator.next();
       BatchReport.Range range = syntaxHighlighting.getRange();
       if (range.getStartLine() <= line) {
-        RangeHelper.appendRange(highlighting, syntaxHighlighting.getRange(), line, lineBuilder.getSource().length());
-        highlighting.append(getCssClass(syntaxHighlighting.getType()));
-        if (range.getEndLine() == line) {
+        String offsets = RangeOffsetHelper.offsetToString(syntaxHighlighting.getRange(), line, lineBuilder.getSource().length());
+        if (!offsets.isEmpty()) {
+          if (highlighting.length() > 0) {
+            highlighting.append(RangeOffsetHelper.SYMBOLS_SEPARATOR);
+          }
+          highlighting.append(offsets)
+            .append(RangeOffsetHelper.OFFSET_SEPARATOR)
+            .append(getCssClass(syntaxHighlighting.getType()));
+          if (range.getEndLine() == line) {
+            syntaxHighlightingIterator.remove();
+          }
+        } else {
           syntaxHighlightingIterator.remove();
         }
       }

server/sonar-server/src/main/java/org/sonar/server/computation/source/RangeHelper.java → server/sonar-server/src/main/java/org/sonar/server/computation/source/RangeOffsetHelper.java

@@ -22,35 +22,31 @@ package org.sonar.server.computation.source;
 
 import org.sonar.batch.protocol.output.BatchReport;
 
-public class RangeHelper {
+public class RangeOffsetHelper {
 
-  private static final String OFFSET_SEPARATOR = ",";
-  private static final String SYMBOLS_SEPARATOR = ";";
+  static final String OFFSET_SEPARATOR = ",";
+  static final String SYMBOLS_SEPARATOR = ";";
 
-  private RangeHelper() {
+  private RangeOffsetHelper() {
     // Only static methods
   }
 
-  public static void appendRange(StringBuilder element, BatchReport.Range range, int lineIndex, int lineLength) {
+  public static String offsetToString(BatchReport.Range range, int lineIndex, int lineLength) {
+    StringBuilder element = new StringBuilder();
+
     validateOffsetOrder(range, lineIndex);
+    validateStartOffsetNotGreaterThanLineLength(range, lineLength, lineIndex);
+    validateEndOffsetNotGreaterThanLineLength(range, lineLength, lineIndex);
 
-    if (element.length() > 0) {
-      element.append(SYMBOLS_SEPARATOR);
-    }
+    int startOffset = range.getStartLine() == lineIndex ? range.getStartOffset() : 0;
+    int endOffset = range.getEndLine() == lineIndex ? range.getEndOffset() : lineLength;
 
-    if (range.getStartLine() == lineIndex) {
-      validateStartOffsetNotGreaterThanLineLength(range, lineLength, lineIndex);
-      element.append(range.getStartOffset()).append(OFFSET_SEPARATOR);
-    } else if (range.getStartLine() < lineIndex) {
-      element.append(0).append(OFFSET_SEPARATOR);
+    if (startOffset < endOffset) {
+      element.append(startOffset).append(OFFSET_SEPARATOR);
+      element.append(endOffset);
     }
 
-    if (range.getEndLine() == lineIndex) {
-      validateEndOffsetNotGreaterThanLineLength(range, lineLength, lineIndex);
-      element.append(range.getEndOffset()).append(OFFSET_SEPARATOR);
-    } else if (range.getEndLine() > lineIndex) {
-      element.append(lineLength - 1).append(OFFSET_SEPARATOR);
-    }
+    return element.toString();
   }
 
   private static void validateOffsetOrder(BatchReport.Range range, int line) {
@@ -60,13 +56,13 @@ public class RangeHelper {
   }
 
   private static void validateStartOffsetNotGreaterThanLineLength(BatchReport.Range range, int lineLength, int line) {
-    if (range.getStartOffset() > lineLength) {
+    if (range.getStartLine() == line && range.getStartOffset() > lineLength) {
       throw new IllegalArgumentException(String.format("Start offset %s is defined outside the length (%s) of the line %s", range.getStartOffset(), lineLength, line));
     }
   }
 
   private static void validateEndOffsetNotGreaterThanLineLength(BatchReport.Range range, int lineLength, int line) {
-    if (range.getEndOffset() > lineLength) {
+    if (range.getEndLine() == line && range.getEndOffset() > lineLength) {
       throw new IllegalArgumentException(String.format("End offset %s is defined outside the length (%s) of the line %s", range.getEndOffset(), lineLength, line));
     }
   }

server/sonar-server/src/main/java/org/sonar/server/computation/source/SymbolsLineReader.java

@@ -24,13 +24,11 @@ import org.sonar.batch.protocol.output.BatchReport;
 import org.sonar.server.source.db.FileSourceDb;
 
 import java.io.Serializable;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import static com.google.common.collect.Lists.newArrayList;
 import static com.google.common.collect.Maps.newHashMap;
+import static com.google.common.collect.Sets.newHashSet;
 
 public class SymbolsLineReader implements LineReader {
 
@@ -38,12 +36,12 @@ public class SymbolsLineReader implements LineReader {
   private final Map<BatchReport.Symbols.Symbol, Integer> idsBySymbol;
 
   public SymbolsLineReader(List<BatchReport.Symbols.Symbol> symbols) {
-    this.symbols = symbols;
+    this.symbols = newArrayList(symbols);
     // Sort symbols to have deterministic results and avoid false variation that would lead to an unnecessary update of the source files
     // data
     Collections.sort(this.symbols, new SymbolsDuplication());
 
-    this.idsBySymbol = createIdsBySymbolMap(symbols);
+    this.idsBySymbol = createIdsBySymbolMap(this.symbols);
   }
 
   @Override
@@ -67,20 +65,30 @@ public class SymbolsLineReader implements LineReader {
 
   private void appendSymbol(StringBuilder lineSymbol, BatchReport.Range range, int line, int symbolId, String sourceLine) {
     if (matchLine(range, line)) {
-      RangeHelper.appendRange(lineSymbol, range, line, sourceLine.length());
-      lineSymbol.append(symbolId);
+      String offsets = RangeOffsetHelper.offsetToString(range, line, sourceLine.length());
+      if (!offsets.isEmpty()) {
+        if (lineSymbol.length() > 0) {
+          lineSymbol.append(RangeOffsetHelper.SYMBOLS_SEPARATOR);
+        }
+        lineSymbol.append(offsets)
+          .append(RangeOffsetHelper.OFFSET_SEPARATOR)
+          .append(symbolId);
+      }
     }
   }
 
   private List<BatchReport.Symbols.Symbol> findSymbolsMatchingLine(int line) {
     List<BatchReport.Symbols.Symbol> lineSymbols = newArrayList();
+    Set<BatchReport.Symbols.Symbol> symbolsIndex = newHashSet();
     for (BatchReport.Symbols.Symbol symbol : symbols) {
-      if (matchLine(symbol.getDeclaration(), line)) {
+      if (matchLine(symbol.getDeclaration(), line) && !symbolsIndex.contains(symbol)) {
         lineSymbols.add(symbol);
+        symbolsIndex.add(symbol);
       } else {
         for (BatchReport.Range range : symbol.getReferenceList()) {
-          if (matchLine(range, line)) {
+          if (matchLine(range, line) && !symbolsIndex.contains(symbol)) {
             lineSymbols.add(symbol);
+            symbolsIndex.add(symbol);
           }
         }
       }

server/sonar-server/src/main/java/org/sonar/server/computation/step/ComputationSteps.java

@@ -47,9 +47,7 @@ public class ComputationSteps {
       PersistComponentLinksStep.class,
       PersistEventsStep.class,
       PersistDuplicationMeasuresStep.class,
-
-      // TODO File sources persistence should not be activated as long as all data are not persisted and persistence should be removed from batch
-//      PersistFileSourcesStep.class,
+      PersistFileSourcesStep.class,
 
       // Switch snapshot and purge
       SwitchSnapshotStep.class,

server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java

@@ -196,7 +196,7 @@ public class PersistFileSourcesStep implements ComputationStep {
         lineReaders.add(new DuplicationLineReader(duplications));
       }
       if (!symbols.isEmpty()) {
-        lineReaders.add(new SymbolsLineReader(newArrayList(symbols)));
+        lineReaders.add(new SymbolsLineReader(symbols));
       }
     }
 

server/sonar-server/src/test/java/org/sonar/server/computation/source/ComputeFileSourceDataTest.java

@@ -38,8 +38,8 @@ public class ComputeFileSourceDataTest {
       );
 
     ComputeFileSourceData.Data data = computeFileSourceData.compute();
-    assertThat(data.getSrcHash()).isEqualTo("1ddab9058a07abc0db2605ab02a61a00");
-    assertThat(data.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\n");
+    assertThat(data.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b");
+    assertThat(data.getSrcHash()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b");
     assertThat(data.getFileSourceData().getLinesList()).hasSize(1);
     assertThat(data.getFileSourceData().getLines(0).getHighlighting()).isEqualTo("h-1");
   }
@@ -53,8 +53,8 @@ public class ComputeFileSourceDataTest {
     );
 
     ComputeFileSourceData.Data data = computeFileSourceData.compute();
-    assertThat(data.getSrcHash()).isEqualTo("4fcc82a88ee38e0aa16c17f512c685c9");
-    assertThat(data.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\ne6251bcf1a7dc3ba5e7933e325bbe605\n");
+    assertThat(data.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\ne6251bcf1a7dc3ba5e7933e325bbe605");
+    assertThat(data.getSrcHash()).isEqualTo("ee5a58024a155466b43bc559d953e018");
     assertThat(data.getFileSourceData().getLinesList()).hasSize(2);
     assertThat(data.getFileSourceData().getLines(0).getHighlighting()).isEqualTo("h-1");
     assertThat(data.getFileSourceData().getLines(1).getHighlighting()).isEqualTo("h-2");
@@ -70,8 +70,8 @@ public class ComputeFileSourceDataTest {
     );
 
     ComputeFileSourceData.Data data = computeFileSourceData.compute();
-    assertThat(data.getSrcHash()).isEqualTo("990c2680400ef07240a32901c101768b");
-    assertThat(data.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\n\n");
+    assertThat(data.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\n");
+    assertThat(data.getSrcHash()).isEqualTo("1ddab9058a07abc0db2605ab02a61a00");
     assertThat(data.getFileSourceData().getLinesList()).hasSize(2);
     assertThat(data.getFileSourceData().getLines(0).getHighlighting()).isEqualTo("h-1");
     assertThat(data.getFileSourceData().getLines(1).getHighlighting()).isEqualTo("h-2");
@@ -98,7 +98,7 @@ public class ComputeFileSourceDataTest {
       newArrayList("   ").iterator(),
       Lists.<LineReader>newArrayList(new MockLineReader()),
       1
-    ).compute().getLineHashes()).isEqualTo("\n");
+    ).compute().getLineHashes()).isEqualTo("");
   }
 
   private static class MockLineReader implements LineReader {

server/sonar-server/src/test/java/org/sonar/server/computation/source/HighlightingLineReaderTest.java

@@ -33,6 +33,12 @@ import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown;
 
 public class HighlightingLineReaderTest {
 
+  FileSourceDb.Data.Builder sourceData = FileSourceDb.Data.newBuilder();
+  FileSourceDb.Line.Builder line1 = sourceData.addLinesBuilder().setSource("line1").setLine(1);
+  FileSourceDb.Line.Builder line2 = sourceData.addLinesBuilder().setSource("line2").setLine(2);
+  FileSourceDb.Line.Builder line3 = sourceData.addLinesBuilder().setSource("line3").setLine(3);
+  FileSourceDb.Line.Builder line4 = sourceData.addLinesBuilder().setSource("line4").setLine(4);
+
   @Test
   public void nothing_to_read() {
     HighlightingLineReader highlightingLineReader = new HighlightingLineReader(Collections.<BatchReport.SyntaxHighlighting>emptyList().iterator());
@@ -55,10 +61,9 @@ public class HighlightingLineReaderTest {
         .build()
     ).iterator());
 
-    FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
-    highlightingLineReader.read(lineBuilder);
+    highlightingLineReader.read(line1);
 
-    assertThat(lineBuilder.getHighlighting()).isEqualTo("2,4,a");
+    assertThat(line1.getHighlighting()).isEqualTo("2,4,a");
   }
 
   @Test
@@ -87,12 +92,9 @@ public class HighlightingLineReaderTest {
         .build()
     ).iterator());
 
-    FileSourceDb.Line.Builder line1 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
     highlightingLineReader.read(line1);
-    FileSourceDb.Line.Builder line2 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line2").setLine(2);
     highlightingLineReader.read(line2);
-    highlightingLineReader.read(FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line3").setLine(3));
-    FileSourceDb.Line.Builder line4 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line4").setLine(4);
+    highlightingLineReader.read(line3);
     highlightingLineReader.read(line4);
 
     assertThat(line1.getHighlighting()).isEqualTo("0,4,a");
@@ -119,10 +121,9 @@ public class HighlightingLineReaderTest {
         .build()
     ).iterator());
 
-    FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
-    highlightingLineReader.read(lineBuilder);
+    highlightingLineReader.read(line1);
 
-    assertThat(lineBuilder.getHighlighting()).isEqualTo("2,3,a;4,5,cd");
+    assertThat(line1.getHighlighting()).isEqualTo("2,3,a;4,5,cd");
   }
 
   @Test
@@ -145,10 +146,9 @@ public class HighlightingLineReaderTest {
         .build()
     ).iterator());
 
-    FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
-    highlightingLineReader.read(lineBuilder);
+    highlightingLineReader.read(line1);
 
-    assertThat(lineBuilder.getHighlighting()).isEqualTo("0,4,c;2,3,k");
+    assertThat(line1.getHighlighting()).isEqualTo("0,4,c;2,3,k");
   }
 
   @Test
@@ -164,19 +164,16 @@ public class HighlightingLineReaderTest {
         .build()
     ).iterator());
 
-    FileSourceDb.Line.Builder line1 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
     highlightingLineReader.read(line1);
-    FileSourceDb.Line.Builder line2 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line 2").setLine(2);
+    FileSourceDb.Line.Builder line2 = sourceData.addLinesBuilder().setSource("line 2").setLine(2);
     highlightingLineReader.read(line2);
-    FileSourceDb.Line.Builder line3 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line3").setLine(3);
     highlightingLineReader.read(line3);
 
-    assertThat(line1.getHighlighting()).isEqualTo("3,4,a");
-    assertThat(line2.getHighlighting()).isEqualTo("0,5,a");
+    assertThat(line1.getHighlighting()).isEqualTo("3,5,a");
+    assertThat(line2.getHighlighting()).isEqualTo("0,6,a");
     assertThat(line3.getHighlighting()).isEqualTo("0,2,a");
   }
 
-  // TODO
   @Test
   public void read_many_syntax_highlighting_on_many_lines() {
     HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
@@ -203,21 +200,39 @@ public class HighlightingLineReaderTest {
         .build()
     ).iterator());
 
-    FileSourceDb.Line.Builder line1 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
     highlightingLineReader.read(line1);
-    FileSourceDb.Line.Builder line2 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line2").setLine(2);
     highlightingLineReader.read(line2);
-    FileSourceDb.Line.Builder line3 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line3").setLine(3);
     highlightingLineReader.read(line3);
-    FileSourceDb.Line.Builder line4 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line4").setLine(4);
     highlightingLineReader.read(line4);
 
-    assertThat(line1.getHighlighting()).isEqualTo("3,4,a");
-    assertThat(line2.getHighlighting()).isEqualTo("0,4,a;0,4,s;1,2,cd");
-    assertThat(line3.getHighlighting()).isEqualTo("0,2,a;0,4,s");
+    assertThat(line1.getHighlighting()).isEqualTo("3,5,a");
+    assertThat(line2.getHighlighting()).isEqualTo("0,5,a;0,5,s;1,2,cd");
+    assertThat(line3.getHighlighting()).isEqualTo("0,2,a;0,5,s");
     assertThat(line4.getHighlighting()).isEqualTo("0,3,s");
   }
 
+  @Test
+  public void read_highlighting_declared_on_a_whole_line() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(2)
+          .setStartOffset(0).setEndOffset(0)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build()
+    ).iterator());
+
+    highlightingLineReader.read(line1);
+    highlightingLineReader.read(line2);
+    highlightingLineReader.read(line3);
+
+    assertThat(line1.getHighlighting()).isEqualTo("0,5,a");
+    // Nothing should be set on line 2
+    assertThat(line2.getHighlighting()).isEmpty();
+    assertThat(line3.getHighlighting()).isEmpty();
+  }
+
   @Test
   public void fail_when_end_offset_is_before_start_offset() {
     HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
@@ -231,7 +246,7 @@ public class HighlightingLineReaderTest {
     ).iterator());
 
     try {
-      highlightingLineReader.read(FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1));
+      highlightingLineReader.read(line1);
       failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
     } catch (IllegalArgumentException e) {
       assertThat(e).hasMessage("End offset 2 cannot be defined before start offset 4 on line 1");
@@ -251,7 +266,7 @@ public class HighlightingLineReaderTest {
     ).iterator());
 
     try {
-      highlightingLineReader.read(FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1));
+      highlightingLineReader.read(line1);
       failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
     } catch (IllegalArgumentException e) {
       assertThat(e).hasMessage("End offset 10 is defined outside the length (5) of the line 1");
@@ -271,7 +286,7 @@ public class HighlightingLineReaderTest {
     ).iterator());
 
     try {
-      highlightingLineReader.read(FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1));
+      highlightingLineReader.read(line1);
       failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
     } catch (IllegalArgumentException e) {
       assertThat(e).hasMessage("Start offset 10 is defined outside the length (5) of the line 1");

server/sonar-server/src/test/java/org/sonar/server/computation/source/RangeHelperTest.java → server/sonar-server/src/test/java/org/sonar/server/computation/source/RangeOffsetHelperTest.java

@@ -26,70 +26,55 @@ import org.sonar.batch.protocol.output.BatchReport;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
 
-public class RangeHelperTest {
+public class RangeOffsetHelperTest {
 
   @Test
   public void append_range() throws Exception {
-    StringBuilder element = new StringBuilder();
-    RangeHelper.appendRange(element, BatchReport.Range.newBuilder()
+    assertThat(RangeOffsetHelper.offsetToString(BatchReport.Range.newBuilder()
       .setStartLine(1).setEndLine(1)
       .setStartOffset(2).setEndOffset(3)
-      .build(),
-      1, 5);
-    assertThat(element.toString()).isEqualTo("2,3,");
-  }
-
-  @Test
-  public void append_range_om_existing_element() throws Exception {
-    StringBuilder element = new StringBuilder("1,2,a");
-    RangeHelper.appendRange(element, BatchReport.Range.newBuilder()
-      .setStartLine(1).setEndLine(1)
-      .setStartOffset(3).setEndOffset(4)
-      .build(),
-      1, 5);
-    assertThat(element.toString()).isEqualTo("1,2,a;3,4,");
+      .build(), 1, 5)).isEqualTo("2,3");
   }
 
   @Test
   public void append_range_not_finishing_in_current_line() throws Exception {
-    StringBuilder element = new StringBuilder();
-    RangeHelper.appendRange(element, BatchReport.Range.newBuilder()
+    assertThat( RangeOffsetHelper.offsetToString(BatchReport.Range.newBuilder()
       .setStartLine(1).setEndLine(3)
       .setStartOffset(2).setEndOffset(3)
-      .build(),
-      1, 5);
-    assertThat(element.toString()).isEqualTo("2,4,");
+      .build(), 1, 5)).isEqualTo("2,5");
   }
 
   @Test
   public void append_range_that_began_in_previous_line_and_finish_in_current_line() throws Exception {
-    StringBuilder element = new StringBuilder();
-    RangeHelper.appendRange(element, BatchReport.Range.newBuilder()
+    assertThat(RangeOffsetHelper.offsetToString(BatchReport.Range.newBuilder()
       .setStartLine(1).setEndLine(3)
       .setStartOffset(2).setEndOffset(3)
-      .build(),
-      3, 5);
-    assertThat(element.toString()).isEqualTo("0,3,");
+      .build(), 3, 5)).isEqualTo("0,3");
   }
 
   @Test
   public void append_range_that_began_in_previous_line_and_not_finishing_in_current_line() throws Exception {
-    StringBuilder element = new StringBuilder();
-    RangeHelper.appendRange(element, BatchReport.Range.newBuilder()
+    assertThat(RangeOffsetHelper.offsetToString(BatchReport.Range.newBuilder()
       .setStartLine(1).setEndLine(3)
       .setStartOffset(2).setEndOffset(3)
-      .build(),
-      2, 5);
-    assertThat(element.toString()).isEqualTo("0,4,");
+      .build(), 2, 5)).isEqualTo("0,5");
+  }
+
+  @Test
+  public void do_nothing_if_offset_is_empty() throws Exception {
+    assertThat(RangeOffsetHelper.offsetToString(BatchReport.Range.newBuilder()
+      .setStartLine(1).setEndLine(1)
+      .setStartOffset(0).setEndOffset(0)
+      .build(), 1, 5)).isEmpty();
   }
 
   @Test
   public void fail_when_end_offset_is_before_start_offset() {
     try {
-      RangeHelper.appendRange(new StringBuilder(), BatchReport.Range.newBuilder()
-        .setStartLine(1).setEndLine(1)
-        .setStartOffset(4).setEndOffset(2)
-        .build(),
+      RangeOffsetHelper.offsetToString(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(4).setEndOffset(2)
+          .build(),
         1, 5);
       failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
     } catch (IllegalArgumentException e) {
@@ -100,10 +85,10 @@ public class RangeHelperTest {
   @Test
   public void fail_when_end_offset_is_higher_than_line_length() {
     try {
-      RangeHelper.appendRange(new StringBuilder(), BatchReport.Range.newBuilder()
-        .setStartLine(1).setEndLine(1)
-        .setStartOffset(4).setEndOffset(10)
-        .build(),
+      RangeOffsetHelper.offsetToString(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(4).setEndOffset(10)
+          .build(),
         1, 5);
       failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
     } catch (IllegalArgumentException e) {
@@ -114,10 +99,10 @@ public class RangeHelperTest {
   @Test
   public void fail_when_start_offset_is_higher_than_line_length() {
     try {
-      RangeHelper.appendRange(new StringBuilder(), BatchReport.Range.newBuilder()
-        .setStartLine(1).setEndLine(1)
-        .setStartOffset(10).setEndOffset(11)
-        .build(),
+      RangeOffsetHelper.offsetToString(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(10).setEndOffset(11)
+          .build(),
         1, 5);
       failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
     } catch (IllegalArgumentException e) {

server/sonar-server/src/test/java/org/sonar/server/computation/source/SymbolsLineReaderTest.java

@@ -115,6 +115,30 @@ public class SymbolsLineReaderTest {
     assertThat(line3.getSymbols()).isEqualTo("1,3,1");
   }
 
+  @Test
+  public void read_symbols_with_two_references_on_the_same_line() throws Exception {
+    List<BatchReport.Symbols.Symbol> symbols = newArrayList(
+      BatchReport.Symbols.Symbol.newBuilder()
+        .setDeclaration(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1).setStartOffset(2).setEndOffset(3)
+          .build())
+        .addReference(BatchReport.Range.newBuilder()
+          .setStartLine(2).setEndLine(2).setStartOffset(0).setEndOffset(1)
+          .build())
+        .addReference(BatchReport.Range.newBuilder()
+          .setStartLine(2).setEndLine(2).setStartOffset(2).setEndOffset(3)
+          .build())
+        .build()
+    );
+
+    SymbolsLineReader symbolsLineReader = new SymbolsLineReader(symbols);
+    symbolsLineReader.read(line1);
+    symbolsLineReader.read(line2);
+
+    assertThat(line1.getSymbols()).isEqualTo("2,3,1");
+    assertThat(line2.getSymbols()).isEqualTo("0,1,1;2,3,1");
+  }
+
   @Test
   public void read_symbols_when_reference_line_is_before_declaration_line() throws Exception {
     List<BatchReport.Symbols.Symbol> symbols = newArrayList(
@@ -247,10 +271,35 @@ public class SymbolsLineReaderTest {
     symbolsLineReader.read(line3);
     symbolsLineReader.read(line4);
 
-    assertThat(line1.getSymbols()).isEqualTo("1,4,1");
+    assertThat(line1.getSymbols()).isEqualTo("1,5,1");
     assertThat(line2.getSymbols()).isEqualTo("0,3,1");
-    assertThat(line3.getSymbols()).isEqualTo("1,4,1");
+    assertThat(line3.getSymbols()).isEqualTo("1,5,1");
     assertThat(line4.getSymbols()).isEqualTo("0,3,1");
   }
 
+  @Test
+  public void read_symbols_declared_on_a_whole_line() throws Exception {
+    List<BatchReport.Symbols.Symbol> symbols = newArrayList(
+      BatchReport.Symbols.Symbol.newBuilder()
+        .setDeclaration(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(2).setStartOffset(0).setEndOffset(0)
+          .build())
+        .addReference(BatchReport.Range.newBuilder()
+          .setStartLine(3).setEndLine(3).setStartOffset(1).setEndOffset(3)
+          .build())
+        .build()
+    );
+
+    SymbolsLineReader symbolsLineReader = new SymbolsLineReader(symbols);
+    symbolsLineReader.read(line1);
+    symbolsLineReader.read(line2);
+    symbolsLineReader.read(line3);
+    symbolsLineReader.read(line4);
+
+    assertThat(line1.getSymbols()).isEqualTo("0,5,1");
+    assertThat(line2.getSymbols()).isEmpty();
+    assertThat(line3.getSymbols()).isEqualTo("1,3,1");
+    assertThat(line4.getSymbols()).isEmpty();
+  }
+
 }

server/sonar-server/src/test/java/org/sonar/server/computation/step/ComputationStepsTest.java

@@ -47,12 +47,13 @@ public class ComputationStepsTest {
       mock(PersistMeasuresStep.class),
       mock(PersistEventsStep.class),
       mock(PersistDuplicationMeasuresStep.class),
-      mock(PersistNumberOfDaysSinceLastCommitStep.class)
+      mock(PersistNumberOfDaysSinceLastCommitStep.class),
+      mock(PersistFileSourcesStep.class)
       );
 
-    assertThat(registry.orderedSteps()).hasSize(16);
+    assertThat(registry.orderedSteps()).hasSize(17);
     assertThat(registry.orderedSteps().get(0)).isInstanceOf(ParseReportStep.class);
-    assertThat(registry.orderedSteps().get(15)).isInstanceOf(SendIssueNotificationsStep.class);
+    assertThat(registry.orderedSteps().get(16)).isInstanceOf(SendIssueNotificationsStep.class);
   }
 
   @Test

server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java

@@ -163,8 +163,8 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
 
     assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1);
     FileSourceDto fileSourceDto = dbClient.fileSourceDao().select("FILE");
-    assertThat(fileSourceDto.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\ne6251bcf1a7dc3ba5e7933e325bbe605\n");
-    assertThat(fileSourceDto.getSrcHash()).isEqualTo("4fcc82a88ee38e0aa16c17f512c685c9");
+    assertThat(fileSourceDto.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\ne6251bcf1a7dc3ba5e7933e325bbe605");
+    assertThat(fileSourceDto.getSrcHash()).isEqualTo("ee5a58024a155466b43bc559d953e018");
   }
 
   @Test
@@ -313,8 +313,8 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
   public void not_update_sources_when_nothing_has_changed() throws Exception {
     // Existing sources
     long past = 150000L;
-    String srcHash = "1ddab9058a07abc0db2605ab02a61a00";
-    String lineHashes = "137f72c3708c6bd0de00a0e5a69c699b\n";
+    String srcHash = "137f72c3708c6bd0de00a0e5a69c699b";
+    String lineHashes = "137f72c3708c6bd0de00a0e5a69c699b";
     String dataHash = "29f25900140c94db38035128cb6de6a2";
 
     dbClient.fileSourceDao().insert(session, new FileSourceDto()
@@ -385,7 +385,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
       .setProjectUuid(PROJECT_UUID)
       .setFileUuid(FILE_UUID)
       // Source hash is missing, update will be made
-      .setLineHashes("137f72c3708c6bd0de00a0e5a69c699b\n")
+      .setLineHashes("137f72c3708c6bd0de00a0e5a69c699b")
       .setDataHash("29f25900140c94db38035128cb6de6a2")
       .setBinaryData(FileSourceDto.encodeData(FileSourceDb.Data.newBuilder()
         .addLines(FileSourceDb.Line.newBuilder()
@@ -406,7 +406,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
     assertThat(fileSourceDto.getCreatedAt()).isEqualTo(past);
     // Updated at is not updated to not reindex the file source in E/S as the src hash is not indexed
     assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(past);
-    assertThat(fileSourceDto.getSrcHash()).isEqualTo("1ddab9058a07abc0db2605ab02a61a00");
+    assertThat(fileSourceDto.getSrcHash()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b");
   }
 
   private BatchReportWriter initBasicReport(int numberOfLines) throws IOException {

sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java

@@ -21,9 +21,6 @@ package org.sonar.batch.index;
 
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.ibatis.session.ResultContext;
-import org.apache.ibatis.session.ResultHandler;
-import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.fs.internal.FileMetadata;
 import org.sonar.api.batch.fs.internal.FileMetadata.LineHashConsumer;
@@ -39,7 +36,6 @@ import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.Map;
 
 public class SourcePersister implements ScanPersister {
@@ -64,24 +60,24 @@ public class SourcePersister implements ScanPersister {
   @Override
   public void persist() {
     // Don't use batch insert for file_sources since keeping all data in memory can produce OOM for big files
-    try (DbSession session = mybatis.openSession(false)) {
-
-      final Map<String, FileSourceDto> previousDtosByUuid = new HashMap<>();
-      session.select("org.sonar.core.source.db.FileSourceMapper.selectHashesForProject", projectTree.getRootProject().getUuid(), new ResultHandler() {
-        @Override
-        public void handleResult(ResultContext context) {
-          FileSourceDto dto = (FileSourceDto) context.getResultObject();
-          previousDtosByUuid.put(dto.getFileUuid(), dto);
-        }
-      });
-
-      FileSourceMapper mapper = session.getMapper(FileSourceMapper.class);
-      for (InputFile inputFile : inputPathCache.allFiles()) {
-        persist(session, mapper, (DefaultInputFile) inputFile, previousDtosByUuid);
-      }
-    } catch (Exception e) {
-      throw new IllegalStateException("Unable to save file sources", e);
-    }
+//    try (DbSession session = mybatis.openSession(false)) {
+//
+//      final Map<String, FileSourceDto> previousDtosByUuid = new HashMap<>();
+//      session.select("org.sonar.core.source.db.FileSourceMapper.selectHashesForProject", projectTree.getRootProject().getUuid(), new ResultHandler() {
+//        @Override
+//        public void handleResult(ResultContext context) {
+//          FileSourceDto dto = (FileSourceDto) context.getResultObject();
+//          previousDtosByUuid.put(dto.getFileUuid(), dto);
+//        }
+//      });
+//
+//      FileSourceMapper mapper = session.getMapper(FileSourceMapper.class);
+//      for (InputFile inputFile : inputPathCache.allFiles()) {
+//        persist(session, mapper, (DefaultInputFile) inputFile, previousDtosByUuid);
+//      }
+//    } catch (Exception e) {
+//      throw new IllegalStateException("Unable to save file sources", e);
+//    }
 
   }