SONAR-5867 Add symbol references data into file_sources table

server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FileSourceDto.java

@@ -72,6 +72,7 @@ class FileSourceDto {
 
   String[] getSourceData() {
     String highlighting = "";
+    String symbolRefs = "";
     ByteArrayOutputStream output = new ByteArrayOutputStream();
     int line = 0;
     String sourceLine = null;
@@ -85,7 +86,7 @@ class FileSourceDto {
         utHits.get(line), utConditions.get(line), utCoveredConditions.get(line),
         itHits.get(line), itConditions.get(line), itCoveredConditions.get(line),
         overallHits.get(line), overallConditions.get(line), overallCoveredConditions.get(line),
-        highlighting, sourceLine);
+        highlighting, symbolRefs, sourceLine);
     }
     csv.close();
     return new String[] {new String(output.toByteArray(), UTF_8), lineHashes.toString()};

server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after-with-scm.xml

@@ -1,7 +1,7 @@
 <dataset>
 
   <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="1416238020000" updated_at="1414770242000"
-    data="aef12a,alice,2014-04-25T12:34:56+0100,1,4,2,2,5,3,3,6,4,,class Foo {&#13;&#10;abe465,bob,2014-07-25T12:34:56+0100,,,,,,,,,,,  // Empty&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,0,,,0,,,0,,,,}&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,,,,,,,,,,,&#13;&#10;"
+    data="aef12a,alice,2014-04-25T12:34:56+0100,1,4,2,2,5,3,3,6,4,,,class Foo {&#13;&#10;abe465,bob,2014-07-25T12:34:56+0100,,,,,,,,,,,,  // Empty&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,0,,,0,,,0,,,,,}&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,,,,,,,,,,,,&#13;&#10;"
     line_hashes="6a19ce786467960a3a9b0d26383a464a&#10;aab2dbc5fdeaa80b050b1d049ede357c&#10;cbb184dd8e05c9709e5dcaedaa0495cf&#10;&#10;"
     data_hash="" />
 

server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after.xml

@@ -1,7 +1,7 @@
 <dataset>
 
   <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="1416238020000" updated_at="1414770242000"
-    data=",,,,,,,,,,,,,class Foo {&#13;&#10;,,,,,,,,,,,,,  // Empty&#13;&#10;,,,,,,,,,,,,,}&#13;&#10;,,,,,,,,,,,,,&#13;&#10;"
+    data=",,,,,,,,,,,,,,class Foo {&#13;&#10;,,,,,,,,,,,,,,  // Empty&#13;&#10;,,,,,,,,,,,,,,}&#13;&#10;,,,,,,,,,,,,,,&#13;&#10;"
     line_hashes="6a19ce786467960a3a9b0d26383a464a&#10;aab2dbc5fdeaa80b050b1d049ede357c&#10;cbb184dd8e05c9709e5dcaedaa0495cf&#10;&#10;"
     data_hash="" />
 

sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java

@@ -29,7 +29,7 @@ import org.apache.ibatis.session.ResultHandler;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.InputPath;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.highlighting.TypeOfText;
+import org.sonar.api.batch.sensor.symbol.Symbol;
 import org.sonar.api.database.model.Snapshot;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Measure;
@@ -43,6 +43,7 @@ import org.sonar.batch.highlighting.SyntaxHighlightingRule;
 import org.sonar.batch.scan.filesystem.InputPathCache;
 import org.sonar.batch.scan.measure.MeasureCache;
 import org.sonar.batch.source.CodeColorizers;
+import org.sonar.batch.symbol.SymbolData;
 import org.sonar.core.persistence.DbSession;
 import org.sonar.core.persistence.MyBatis;
 import org.sonar.core.source.SnapshotDataTypes;
@@ -221,6 +222,7 @@ public class SourcePersister implements ScanPersister {
     Map<Integer, String> overallCoveredCondByLine = getLineMetric(file, CoreMetrics.OVERALL_COVERED_CONDITIONS_BY_LINE_KEY);
     SyntaxHighlightingData highlighting = loadHighlighting(file);
     String[] highlightingPerLine = computeHighlightingPerLine(file, highlighting);
+    String[] symbolReferencesPerLine = computeSymbolReferencesPerLine(file, loadSymbolReferences(file));
 
     ByteArrayOutputStream output = new ByteArrayOutputStream();
     CsvWriter csv = CsvWriter.of(new OutputStreamWriter(output, UTF_8));
@@ -229,7 +231,7 @@ public class SourcePersister implements ScanPersister {
         utHitsByLine.get(lineIdx), utCondByLine.get(lineIdx), utCoveredCondByLine.get(lineIdx),
         itHitsByLine.get(lineIdx), itCondByLine.get(lineIdx), itCoveredCondByLine.get(lineIdx),
         overallHitsByLine.get(lineIdx), overallCondByLine.get(lineIdx), overallCoveredCondByLine.get(lineIdx),
-        highlightingPerLine[lineIdx - 1],
+        highlightingPerLine[lineIdx - 1], symbolReferencesPerLine[lineIdx - 1],
         CharMatcher.anyOf(BOM).removeFrom(lines.get(lineIdx - 1)));
     }
     csv.close();
@@ -245,6 +247,11 @@ public class SourcePersister implements ScanPersister {
     return highlighting;
   }
 
+  @CheckForNull
+  private SymbolData loadSymbolReferences(DefaultInputFile file) {
+    return componentDataCache.getData(file.key(), SnapshotDataTypes.SYMBOL_HIGHLIGHTING);
+  }
+
   String[] computeHighlightingPerLine(DefaultInputFile file, @Nullable SyntaxHighlightingData highlighting) {
     String[] result = new String[file.lines()];
     if (highlighting == null) {
@@ -259,7 +266,7 @@ public class SourcePersister implements ScanPersister {
         currentLineIdx++;
       }
       // Now we know current rule starts on current line
-      writeRule(file, rule, highlightingPerLine, currentLineIdx);
+      writeDataPerLine(file.originalLineOffsets(), rule, rule.getStartPosition(), rule.getEndPosition(), highlightingPerLine, currentLineIdx, new RuleItemWriter());
     }
     for (int i = 0; i < file.lines(); i++) {
       result[i] = highlightingPerLine[i] != null ? highlightingPerLine[i].toString() : null;
@@ -267,41 +274,115 @@ public class SourcePersister implements ScanPersister {
     return result;
   }
 
-  private void writeRule(DefaultInputFile file, SyntaxHighlightingRule rule, StringBuilder[] highlightingPerLine, int currentLine) {
-    int currentLineIdx = currentLine;
+  String[] computeSymbolReferencesPerLine(DefaultInputFile file, @Nullable SymbolData symbolRefs) {
+    String[] result = new String[file.lines()];
+    if (symbolRefs == null) {
+      return result;
+    }
+    StringBuilder[] symbolRefsPerLine = new StringBuilder[file.lines()];
+    long[] originalLineOffsets = file.originalLineOffsets();
+    int symbolId = 1;
+    for (Symbol symbol : symbolRefs.referencesBySymbol().keySet()) {
+      int declarationStartOffset = symbol.getDeclarationStartOffset();
+      int declarationEndOffset = symbol.getDeclarationEndOffset();
+      int length = declarationEndOffset - declarationStartOffset;
+      addSymbol(symbolId, declarationStartOffset, declarationEndOffset, originalLineOffsets, symbolRefsPerLine);
+      for (Integer referenceStartOffset : symbolRefs.referencesBySymbol().get(symbol)) {
+        if (referenceStartOffset == declarationStartOffset) {
+          // Ignore old API that used to store reference as first declaration
+          continue;
+        }
+        addSymbol(symbolId, referenceStartOffset, referenceStartOffset + length, originalLineOffsets, symbolRefsPerLine);
+      }
+      symbolId++;
+    }
+    for (int i = 0; i < file.lines(); i++) {
+      result[i] = symbolRefsPerLine[i] != null ? symbolRefsPerLine[i].toString() : null;
+    }
+    return result;
+  }
+
+  private void addSymbol(int symbolId, int startOffset, int endOffset, long[] originalLineOffsets, StringBuilder[] result) {
+    int startLine = binarySearchLine(startOffset, originalLineOffsets);
+    writeDataPerLine(originalLineOffsets, symbolId, startOffset, endOffset, result, startLine, new SymbolItemWriter());
+  }
+
+  private int binarySearchLine(int declarationStartOffset, long[] originalLineOffsets) {
+    int begin = 0;
+    int end = originalLineOffsets.length - 1;
+    while (begin < end) {
+      int mid = (int) Math.round((begin + end) / 2D);
+      if (declarationStartOffset < originalLineOffsets[mid]) {
+        end = mid - 1;
+      } else {
+        begin = mid;
+      }
+    }
+    return begin + 1;
+  }
+
+  private <G> void writeDataPerLine(long[] originalLineOffsets, G item, int globalStartOffset, int globalEndOffset, StringBuilder[] dataPerLine, int startLine,
+    RangeItemWriter<G> writer) {
+    int currentLineIdx = startLine;
     // We know current rule starts on current line
-    long ruleStartOffsetCurrentLine = rule.getStartPosition();
-    while (currentLineIdx < file.lines() && rule.getEndPosition() >= file.originalLineOffsets()[currentLineIdx]) {
+    long ruleStartOffsetCurrentLine = globalStartOffset;
+    while (currentLineIdx < originalLineOffsets.length && globalEndOffset >= originalLineOffsets[currentLineIdx]) {
       // rule continue on next line so write current line and continue on next line with same rule
-      writeRule(highlightingPerLine, currentLineIdx, ruleStartOffsetCurrentLine - file.originalLineOffsets()[currentLineIdx - 1], file.originalLineOffsets()[currentLineIdx]
-        - file.originalLineOffsets()[currentLineIdx - 1],
-        rule.getTextType());
+      writeItem(item, dataPerLine, currentLineIdx, ruleStartOffsetCurrentLine - originalLineOffsets[currentLineIdx - 1], originalLineOffsets[currentLineIdx]
+        - originalLineOffsets[currentLineIdx - 1], writer);
       currentLineIdx++;
-      ruleStartOffsetCurrentLine = file.originalLineOffsets()[currentLineIdx - 1];
+      ruleStartOffsetCurrentLine = originalLineOffsets[currentLineIdx - 1];
     }
     // Rule ends on current line
-    writeRule(highlightingPerLine, currentLineIdx, ruleStartOffsetCurrentLine - file.originalLineOffsets()[currentLineIdx - 1], rule.getEndPosition()
-      - file.originalLineOffsets()[currentLineIdx - 1],
-      rule.getTextType());
+    writeItem(item, dataPerLine, currentLineIdx, ruleStartOffsetCurrentLine - originalLineOffsets[currentLineIdx - 1], globalEndOffset
+      - originalLineOffsets[currentLineIdx - 1], writer);
   }
 
-  private void writeRule(StringBuilder[] highlightingPerLine, int currentLineIdx, long startLineOffset, long endLineOffset, TypeOfText textType) {
-    if (highlightingPerLine[currentLineIdx - 1] == null) {
-      highlightingPerLine[currentLineIdx - 1] = new StringBuilder();
+  private <G> void writeItem(G item, StringBuilder[] dataPerLine, int currentLineIdx, long startLineOffset, long endLineOffset, RangeItemWriter<G> writer) {
+    if (dataPerLine[currentLineIdx - 1] == null) {
+      dataPerLine[currentLineIdx - 1] = new StringBuilder();
     }
-    StringBuilder currentLineSb = highlightingPerLine[currentLineIdx - 1];
-    writeRule(currentLineSb, startLineOffset, endLineOffset, textType);
+    StringBuilder currentLineSb = dataPerLine[currentLineIdx - 1];
+    writer.writeItem(currentLineSb, startLineOffset, endLineOffset, item);
   }
 
-  private void writeRule(StringBuilder currentLineSb, long startLineOffset, long endLineOffset, TypeOfText textType) {
-    if (currentLineSb.length() > 0) {
-      currentLineSb.append(SyntaxHighlightingData.RULE_SEPARATOR);
+  private static interface RangeItemWriter<G> {
+    /**
+     * Write item on a single line
+     */
+    void writeItem(StringBuilder currentLineSb, long startLineOffset, long endLineOffset, G item);
+  }
+
+  private static class RuleItemWriter implements RangeItemWriter<SyntaxHighlightingRule> {
+
+    @Override
+    public void writeItem(StringBuilder currentLineSb, long startLineOffset, long endLineOffset, SyntaxHighlightingRule item) {
+      if (currentLineSb.length() > 0) {
+        currentLineSb.append(SyntaxHighlightingData.RULE_SEPARATOR);
+      }
+      currentLineSb.append(startLineOffset)
+        .append(SyntaxHighlightingData.FIELD_SEPARATOR)
+        .append(endLineOffset)
+        .append(SyntaxHighlightingData.FIELD_SEPARATOR)
+        .append(item.getTextType().cssClass());
     }
-    currentLineSb.append(startLineOffset)
-      .append(SyntaxHighlightingData.FIELD_SEPARATOR)
-      .append(endLineOffset)
-      .append(SyntaxHighlightingData.FIELD_SEPARATOR)
-      .append(textType.cssClass());
+
+  }
+
+  private static class SymbolItemWriter implements RangeItemWriter<Integer> {
+
+    @Override
+    public void writeItem(StringBuilder currentLineSb, long startLineOffset, long endLineOffset, Integer symbolId) {
+      if (currentLineSb.length() > 0) {
+        currentLineSb.append(SymbolData.SYMBOL_SEPARATOR);
+      }
+      currentLineSb.append(startLineOffset)
+        .append(SymbolData.FIELD_SEPARATOR)
+        .append(endLineOffset)
+        .append(SymbolData.FIELD_SEPARATOR)
+        .append(symbolId);
+    }
+
   }
 
   private Map<Integer, String> getLineMetric(DefaultInputFile file, String metricKey) {

sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java

@@ -80,7 +80,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.Set;
 
 /**
  * Main utility class for writing batch medium tests.
@@ -407,7 +406,7 @@ public class BatchMediumTester {
      * @param symbolEndOffset 0-based end offset for the symbol in file
      */
     @CheckForNull
-    public Set<Integer> symbolReferencesFor(InputFile file, int symbolStartOffset, int symbolEndOffset) {
+    public List<Integer> symbolReferencesFor(InputFile file, int symbolStartOffset, int symbolEndOffset) {
       SymbolData data = symbolTablePerFile.get(file);
       if (data == null) {
         return null;

sonar-batch/src/main/java/org/sonar/batch/source/DefaultSymbolTable.java

@@ -20,25 +20,24 @@
 
 package org.sonar.batch.source;
 
-import com.google.common.collect.SortedSetMultimap;
-import com.google.common.collect.TreeMultimap;
 import org.sonar.api.batch.sensor.symbol.internal.DefaultSymbol;
 import org.sonar.api.source.Symbol;
 import org.sonar.api.source.Symbolizable;
-import org.sonar.batch.symbol.DefaultSymbolTableBuilder;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 public class DefaultSymbolTable implements Symbolizable.SymbolTable {
 
-  private SortedSetMultimap<org.sonar.api.batch.sensor.symbol.Symbol, Integer> referencesBySymbol;
+  private Map<Symbol, List<Integer>> referencesBySymbol;
 
-  private DefaultSymbolTable(SortedSetMultimap<org.sonar.api.batch.sensor.symbol.Symbol, Integer> referencesBySymbol) {
+  private DefaultSymbolTable(Map<Symbol, List<Integer>> referencesBySymbol) {
     this.referencesBySymbol = referencesBySymbol;
   }
 
-  public SortedSetMultimap<org.sonar.api.batch.sensor.symbol.Symbol, Integer> getReferencesBySymbol() {
+  public Map<Symbol, List<Integer>> getReferencesBySymbol() {
     return referencesBySymbol;
   }
 
@@ -58,27 +57,29 @@ public class DefaultSymbolTable implements Symbolizable.SymbolTable {
 
   public static class Builder implements Symbolizable.SymbolTableBuilder {
 
-    private final SortedSetMultimap<org.sonar.api.batch.sensor.symbol.Symbol, Integer> referencesBySymbol;
+    private final Map<Symbol, List<Integer>> referencesBySymbol = new LinkedHashMap<Symbol, List<Integer>>();
     private final String componentKey;
 
     public Builder(String componentKey) {
       this.componentKey = componentKey;
-      referencesBySymbol = TreeMultimap.create(new DefaultSymbolTableBuilder.SymbolComparator(), new DefaultSymbolTableBuilder.ReferenceComparator());
     }
 
     @Override
     public Symbol newSymbol(int fromOffset, int toOffset) {
-      Symbol symbol = new DefaultSymbol(componentKey, fromOffset, toOffset);
-      referencesBySymbol.put(symbol, symbol.getDeclarationStartOffset());
+      Symbol symbol = new DefaultSymbol(fromOffset, toOffset);
+      referencesBySymbol.put(symbol, new ArrayList<Integer>());
       return symbol;
     }
 
     @Override
     public void newReference(Symbol symbol, int fromOffset) {
+      if (!referencesBySymbol.containsKey(symbol)) {
+        throw new UnsupportedOperationException("Cannot add reference to a symbol in another file");
+      }
       if (fromOffset >= symbol.getDeclarationStartOffset() && fromOffset < symbol.getDeclarationEndOffset()) {
         throw new UnsupportedOperationException("Cannot add reference (" + fromOffset + ") overlapping " + symbol);
       }
-      referencesBySymbol.put(symbol, fromOffset);
+      referencesBySymbol.get(symbol).add(fromOffset);
     }
 
     @Override

sonar-batch/src/main/java/org/sonar/batch/source/DefaultSymbolizable.java

@@ -49,6 +49,6 @@ public class DefaultSymbolizable implements Symbolizable {
   @Override
   public void setSymbolTable(SymbolTable symbolTable) {
     SymbolData symbolData = new SymbolData(((DefaultSymbolTable) symbolTable).getReferencesBySymbol());
-    cache.setStringData(component().key(), SnapshotDataTypes.SYMBOL_HIGHLIGHTING, symbolData.writeString());
+    cache.setData(component().key(), SnapshotDataTypes.SYMBOL_HIGHLIGHTING, symbolData);
   }
 }

sonar-batch/src/main/java/org/sonar/batch/symbol/DefaultSymbolTableBuilder.java

@@ -20,8 +20,6 @@
 
 package org.sonar.batch.symbol;
 
-import com.google.common.collect.SortedSetMultimap;
-import com.google.common.collect.TreeMultimap;
 import org.sonar.api.batch.sensor.symbol.Symbol;
 import org.sonar.api.batch.sensor.symbol.SymbolTableBuilder;
 import org.sonar.api.batch.sensor.symbol.internal.DefaultSymbol;
@@ -29,43 +27,48 @@ import org.sonar.batch.index.ComponentDataCache;
 import org.sonar.core.source.SnapshotDataTypes;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 public class DefaultSymbolTableBuilder implements SymbolTableBuilder {
 
   private final String componentKey;
   private final ComponentDataCache cache;
-  private final SortedSetMultimap<Symbol, Integer> referencesBySymbol;
+  private final Map<org.sonar.api.source.Symbol, List<Integer>> referencesBySymbol = new LinkedHashMap<org.sonar.api.source.Symbol, List<Integer>>();
 
   public DefaultSymbolTableBuilder(String componentKey, ComponentDataCache cache) {
     this.componentKey = componentKey;
     this.cache = cache;
-    this.referencesBySymbol = TreeMultimap.create(new SymbolComparator(), new ReferenceComparator());
   }
 
   @Override
   public Symbol newSymbol(int fromOffset, int toOffset) {
-    Symbol symbol = new DefaultSymbol(componentKey, fromOffset, toOffset);
-    referencesBySymbol.put(symbol, symbol.getDeclarationStartOffset());
+    org.sonar.api.source.Symbol symbol = new DefaultSymbol(fromOffset, toOffset);
+    referencesBySymbol.put(symbol, new ArrayList<Integer>());
     return symbol;
   }
 
   @Override
   public void newReference(Symbol symbol, int fromOffset) {
-    String otherComponentKey = ((DefaultSymbol) symbol).componentKey();
-    if (!otherComponentKey.equals(componentKey)) {
-      throw new UnsupportedOperationException("Cannot add reference from (" + componentKey + ") to another file (" + otherComponentKey + ")");
+    if (!referencesBySymbol.containsKey(symbol)) {
+      throw new UnsupportedOperationException("Cannot add reference to a symbol in another file");
     }
     if (fromOffset >= symbol.getDeclarationStartOffset() && fromOffset < symbol.getDeclarationEndOffset()) {
       throw new UnsupportedOperationException("Cannot add reference (" + fromOffset + ") overlapping " + symbol);
     }
-    referencesBySymbol.put(symbol, fromOffset);
+    referencesBySymbol.get(symbol).add(fromOffset);
+  }
+
+  public SymbolData build() {
+    return new SymbolData(referencesBySymbol);
   }
 
   @Override
   public void done() {
-    SymbolData symbolData = new SymbolData(referencesBySymbol);
-    cache.setData(componentKey, SnapshotDataTypes.SYMBOL_HIGHLIGHTING, symbolData);
+    cache.setData(componentKey, SnapshotDataTypes.SYMBOL_HIGHLIGHTING, build());
   }
 
   public static class SymbolComparator implements Comparator<Symbol>, Serializable {

sonar-batch/src/main/java/org/sonar/batch/symbol/SymbolData.java

@@ -20,24 +20,25 @@
 
 package org.sonar.batch.symbol;
 
-import com.google.common.collect.SortedSetMultimap;
 import org.sonar.api.batch.sensor.symbol.Symbol;
 import org.sonar.batch.index.Data;
 
 import java.util.Collection;
+import java.util.List;
+import java.util.Map;
 
 public class SymbolData implements Data {
 
-  private static final String FIELD_SEPARATOR = ",";
-  private static final String SYMBOL_SEPARATOR = ";";
+  public static final String FIELD_SEPARATOR = ",";
+  public static final String SYMBOL_SEPARATOR = ";";
 
-  private final SortedSetMultimap<Symbol, Integer> referencesBySymbol;
+  private final Map<org.sonar.api.source.Symbol, List<Integer>> referencesBySymbol;
 
-  public SymbolData(SortedSetMultimap<Symbol, Integer> referencesBySymbol) {
+  public SymbolData(Map<org.sonar.api.source.Symbol, List<Integer>> referencesBySymbol) {
     this.referencesBySymbol = referencesBySymbol;
   }
 
-  public SortedSetMultimap<Symbol, Integer> referencesBySymbol() {
+  public Map<org.sonar.api.source.Symbol, List<Integer>> referencesBySymbol() {
     return referencesBySymbol;
   }
 
@@ -46,15 +47,19 @@ public class SymbolData implements Data {
     StringBuilder sb = new StringBuilder();
 
     for (Symbol symbol : referencesBySymbol.keySet()) {
+      if (sb.length() > 0) {
+        sb.append(SYMBOL_SEPARATOR);
+      }
 
       sb.append(symbol.getDeclarationStartOffset())
         .append(FIELD_SEPARATOR)
-        .append(symbol.getDeclarationEndOffset());
+        .append(symbol.getDeclarationEndOffset())
+        .append(FIELD_SEPARATOR)
+        .append(symbol.getDeclarationStartOffset());
       Collection<Integer> symbolReferences = referencesBySymbol.get(symbol);
       for (Integer symbolReference : symbolReferences) {
         sb.append(FIELD_SEPARATOR).append(symbolReference);
       }
-      sb.append(SYMBOL_SEPARATOR);
     }
 
     return sb.toString();

sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java

@@ -42,6 +42,7 @@ import org.sonar.batch.highlighting.SyntaxHighlightingDataBuilder;
 import org.sonar.batch.scan.filesystem.InputPathCache;
 import org.sonar.batch.scan.measure.MeasureCache;
 import org.sonar.batch.source.CodeColorizers;
+import org.sonar.batch.symbol.DefaultSymbolTableBuilder;
 import org.sonar.core.persistence.AbstractDaoTestCase;
 import org.sonar.core.source.SnapshotDataTypes;
 import org.sonar.core.source.db.FileSourceDao;
@@ -145,9 +146,9 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     assertThat(fileSourceDto.getCreatedAt()).isEqualTo(DateUtils.parseDateTime("2014-10-10T16:44:02+0200").getTime());
     assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getData()).isEqualTo(
-      ",,,,,,,,,,,,,changed\r\n,,,,,,,,,,,,,content\r\n");
+      ",,,,,,,,,,,,,,changed\r\n,,,,,,,,,,,,,,content\r\n");
     assertThat(fileSourceDto.getLineHashes()).isEqualTo(md5Hex("changed") + "\n" + md5Hex("content"));
-    assertThat(fileSourceDto.getDataHash()).isEqualTo("00141b1194a360a5d5d1f9dcffb27359");
+    assertThat(fileSourceDto.getDataHash()).isEqualTo("bd582d7001cfca180c3dacab10043292");
   }
 
   @Test
@@ -189,9 +190,9 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getData()).isEqualTo(
-      ",,,,,,,,,,,,,foo\r\n,,,,,,,,,,,,,bar\r\n,,,,,,,,,,,,,biz\r\n");
+      ",,,,,,,,,,,,,,foo\r\n,,,,,,,,,,,,,,bar\r\n,,,,,,,,,,,,,,biz\r\n");
     assertThat(fileSourceDto.getLineHashes()).isEqualTo(md5Hex("foo") + "\n" + md5Hex("bar") + "\n" + md5Hex("biz"));
-    assertThat(fileSourceDto.getDataHash()).isEqualTo("e6860232a097eb0616b9fe1bad760941");
+    assertThat(fileSourceDto.getDataHash()).isEqualTo("e1827ac156bb76144486e6570a591cfb");
 
   }
 
@@ -246,6 +247,16 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     when(componentDataCache.getData(PROJECT_KEY + ":" + relativePathNew, SnapshotDataTypes.SYNTAX_HIGHLIGHTING))
       .thenReturn(highlighting);
 
+    DefaultSymbolTableBuilder symbolBuilder = new DefaultSymbolTableBuilder(PROJECT_KEY + ":" + relativePathNew, null);
+    org.sonar.api.batch.sensor.symbol.Symbol s1 = symbolBuilder.newSymbol(1, 2);
+    symbolBuilder.newReference(s1, 4);
+    symbolBuilder.newReference(s1, 11);
+    org.sonar.api.batch.sensor.symbol.Symbol s2 = symbolBuilder.newSymbol(4, 6);
+    symbolBuilder.newReference(s2, 0);
+    symbolBuilder.newReference(s2, 7);
+    when(componentDataCache.getData(PROJECT_KEY + ":" + relativePathNew, SnapshotDataTypes.SYMBOL_HIGHLIGHTING))
+      .thenReturn(symbolBuilder.build());
+
     sourcePersister.persist();
 
     FileSourceDto fileSourceDto = new FileSourceDao(getMyBatis()).select("uuidnew");
@@ -253,10 +264,10 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getLineHashes()).isEqualTo(md5Hex("foo") + "\n" + md5Hex("bar") + "\n" + md5Hex("biz"));
     assertThat(fileSourceDto.getData()).isEqualTo(
-      "123,julien,2014-10-11T16:44:02+0100,1,4,2,2,5,3,3,6,4,\"0,3,a\",foo\r\n"
-        + "234,simon,2014-10-12T16:44:02+0100,,,,,,,,,,\"0,1,cd\",bar\r\n"
-        + "345,julien,2014-10-13T16:44:02+0100,0,,,0,,,0,,,\"0,9,c\",biz\r\n");
-    assertThat(fileSourceDto.getDataHash()).isEqualTo("cb7bdbb98bd053c7367c92e6596b37c0");
+      "123,julien,2014-10-11T16:44:02+0100,1,4,2,2,5,3,3,6,4,\"0,3,a\",\"1,2,1;0,2,2\",foo\r\n"
+        + "234,simon,2014-10-12T16:44:02+0100,,,,,,,,,,\"0,1,cd\",\"0,1,1;0,2,2\",bar\r\n"
+        + "345,julien,2014-10-13T16:44:02+0100,0,,,0,,,0,,,\"0,9,c\",\"4,5,1;0,2,2\",biz\r\n");
+    assertThat(fileSourceDto.getDataHash()).isEqualTo("594752666dd282f4a3bb985829c790fa");
   }
 
   @Test
@@ -329,6 +340,25 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     assertThat(highlightingPerLine).containsOnly("0,3,a", "0,3,c;0,2,cd", "0,9,c;1,8,k");
   }
 
+  @Test
+  public void testSimpleConversionOfSymbolOffset() {
+    DefaultInputFile file = new DefaultInputFile(PROJECT_KEY, "src/foo.java")
+      .setLines(3)
+      .setOriginalLineOffsets(new long[] {0, 4, 7});
+
+    DefaultSymbolTableBuilder symbolBuilder = new DefaultSymbolTableBuilder(PROJECT_KEY + ":" + "src/foo.java", null);
+    org.sonar.api.batch.sensor.symbol.Symbol s1 = symbolBuilder.newSymbol(1, 2);
+    symbolBuilder.newReference(s1, 4);
+    symbolBuilder.newReference(s1, 11);
+    org.sonar.api.batch.sensor.symbol.Symbol s2 = symbolBuilder.newSymbol(4, 6);
+    symbolBuilder.newReference(s2, 0);
+    symbolBuilder.newReference(s2, 7);
+
+    String[] symbolsPerLine = sourcePersister.computeSymbolReferencesPerLine(file, symbolBuilder.build());
+
+    assertThat(symbolsPerLine).containsOnly("1,2,1;0,2,2", "0,1,1;0,2,2", "4,5,1;0,2,2");
+  }
+
   private void mockResourceCache(String relativePathEmpty, String projectKey, String uuid) {
     File sonarFile = File.create(relativePathEmpty);
     sonarFile.setUuid(uuid);

sonar-batch/src/test/java/org/sonar/batch/mediumtest/symbol/SymbolMediumTest.java

@@ -81,7 +81,7 @@ public class SymbolMediumTest {
       .start();
 
     InputFile file = result.inputFile("src/sample.xoo");
-    assertThat(result.symbolReferencesFor(file, 7, 10)).containsOnly(7, 27);
+    assertThat(result.symbolReferencesFor(file, 7, 10)).containsOnly(27);
   }
 
 }

sonar-batch/src/test/java/org/sonar/batch/source/DefaultSymbolTableTest.java

@@ -44,10 +44,10 @@ public class DefaultSymbolTableTest {
     symbolTableBuilder.newReference(thirdSymbol, 70);
     Symbolizable.SymbolTable symbolTable = symbolTableBuilder.build();
 
-    assertThat(symbolTable.symbols()).containsExactly(firstSymbol, thirdSymbol, secondSymbol);
-    assertThat(symbolTable.references(firstSymbol)).containsExactly(10, 32);
-    assertThat(symbolTable.references(secondSymbol)).containsExactly(84, 124);
-    assertThat(symbolTable.references(thirdSymbol)).containsExactly(55, 70);
+    assertThat(symbolTable.symbols()).containsExactly(firstSymbol, secondSymbol, thirdSymbol);
+    assertThat(symbolTable.references(firstSymbol)).containsExactly(32);
+    assertThat(symbolTable.references(secondSymbol)).containsExactly(124);
+    assertThat(symbolTable.references(thirdSymbol)).containsExactly(70);
   }
 
   @Test
@@ -64,6 +64,6 @@ public class DefaultSymbolTableTest {
     Symbolizable.SymbolTableBuilder symbolTableBuilder = new DefaultSymbolTable.Builder("foo");
     Symbol symbol = symbolTableBuilder.newSymbol(10, 20);
 
-    assertThat(symbol.toString()).isEqualTo("Symbol{component=foo, offset=10-20}");
+    assertThat(symbol.toString()).isEqualTo("Symbol{offset=10-20}");
   }
 }

sonar-batch/src/test/java/org/sonar/batch/source/DefaultSymbolizableTest.java

@@ -25,9 +25,14 @@ import org.sonar.api.component.Component;
 import org.sonar.api.source.Symbol;
 import org.sonar.api.source.Symbolizable;
 import org.sonar.batch.index.ComponentDataCache;
+import org.sonar.batch.symbol.SymbolData;
 import org.sonar.core.source.SnapshotDataTypes;
 
-import static org.mockito.Mockito.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class DefaultSymbolizableTest {
 
@@ -52,6 +57,6 @@ public class DefaultSymbolizableTest {
 
     symbolPerspective.setSymbolTable(symbolTable);
 
-    verify(cache).setStringData("myComponent", SnapshotDataTypes.SYMBOL_HIGHLIGHTING, "4,8,4,12,70;25,33,25,44,60,108;");
+    verify(cache).setData(eq("myComponent"), eq(SnapshotDataTypes.SYMBOL_HIGHLIGHTING), any(SymbolData.class));
   }
 }

sonar-batch/src/test/java/org/sonar/batch/symbol/DefaultSymbolTableBuilderTest.java

@@ -20,7 +20,6 @@
 
 package org.sonar.batch.symbol;
 
-import com.google.common.collect.SortedSetMultimap;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -31,6 +30,8 @@ import org.sonar.batch.index.ComponentDataCache;
 import org.sonar.core.source.SnapshotDataTypes;
 
 import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 import static org.fest.assertions.Assertions.assertThat;
 import static org.mockito.Matchers.eq;
@@ -43,7 +44,7 @@ public class DefaultSymbolTableBuilderTest {
   public ExpectedException throwable = ExpectedException.none();
 
   @Test
-  public void should_order_symbol_and_references() throws Exception {
+  public void should_write_symbol_and_references() throws Exception {
     ComponentDataCache componentDataCache = mock(ComponentDataCache.class);
     SymbolTableBuilder symbolTableBuilder = new DefaultSymbolTableBuilder("foo", componentDataCache);
     Symbol firstSymbol = symbolTableBuilder.newSymbol(10, 20);
@@ -57,14 +58,14 @@ public class DefaultSymbolTableBuilderTest {
     ArgumentCaptor<SymbolData> argCaptor = ArgumentCaptor.forClass(SymbolData.class);
     verify(componentDataCache).setData(eq("foo"), eq(SnapshotDataTypes.SYMBOL_HIGHLIGHTING), argCaptor.capture());
 
-    SortedSetMultimap<Symbol, Integer> referencesBySymbol = argCaptor.getValue().referencesBySymbol();
+    Map<org.sonar.api.source.Symbol, List<Integer>> referencesBySymbol = argCaptor.getValue().referencesBySymbol();
 
-    assertThat(new ArrayList<Symbol>(referencesBySymbol.keySet())).containsExactly(firstSymbol, thirdSymbol, secondSymbol);
-    assertThat(new ArrayList<Integer>(referencesBySymbol.get(firstSymbol))).containsExactly(10, 32);
-    assertThat(new ArrayList<Integer>(referencesBySymbol.get(secondSymbol))).containsExactly(84, 124);
-    assertThat(new ArrayList<Integer>(referencesBySymbol.get(thirdSymbol))).containsExactly(55, 70);
+    assertThat(new ArrayList<Symbol>(referencesBySymbol.keySet())).containsExactly(firstSymbol, secondSymbol, thirdSymbol);
+    assertThat(new ArrayList<Integer>(referencesBySymbol.get(firstSymbol))).containsExactly(32);
+    assertThat(new ArrayList<Integer>(referencesBySymbol.get(secondSymbol))).containsExactly(124);
+    assertThat(new ArrayList<Integer>(referencesBySymbol.get(thirdSymbol))).containsExactly(70);
 
-    assertThat(argCaptor.getValue().writeString()).isEqualTo("10,20,10,32;55,62,55,70;84,92,84,124;");
+    assertThat(argCaptor.getValue().writeString()).isEqualTo("10,20,10,32;84,92,84,124;55,62,55,70");
   }
 
   @Test
@@ -78,7 +79,7 @@ public class DefaultSymbolTableBuilderTest {
     ArgumentCaptor<SymbolData> argCaptor = ArgumentCaptor.forClass(SymbolData.class);
     verify(componentDataCache).setData(eq("foo"), eq(SnapshotDataTypes.SYMBOL_HIGHLIGHTING), argCaptor.capture());
 
-    assertThat(argCaptor.getValue().writeString()).isEqualTo("10,20,10;");
+    assertThat(argCaptor.getValue().writeString()).isEqualTo("10,20,10");
   }
 
   @Test

sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml

@@ -1,8 +1,8 @@
 <dataset>
   <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" 
-      data=",,,,,,,,,,,,,unchanged&#13;&#10;,,,,,,,,,,,,,content&#13;&#10;" 
+      data=",,,,,,,,,,,,,,unchanged&#13;&#10;,,,,,,,,,,,,,,content&#13;&#10;" 
       line_hashes="8d7b3d6b83c0a517eac07e1aac94b773&#10;9a0364b9e99bb480dd25e1f0284c8555" 
-      data_hash="4fc300c67452d04e8852cbc315e16010" 
+      data_hash="21a2d025d55b25d6412b1565afb516a5" 
       created_at="1412952242000" updated_at="1412952242000" />
       
 </dataset>

sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml

@@ -1,9 +1,9 @@
 <dataset>
 
   <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" 
-      data=",,,,,,,,,,,,,unchanged&#13;&#10;,,,,,,,,,,,,,content&#13;&#10;" 
+      data=",,,,,,,,,,,,,,unchanged&#13;&#10;,,,,,,,,,,,,,,content&#13;&#10;" 
       line_hashes="8d7b3d6b83c0a517eac07e1aac94b773&#10;9a0364b9e99bb480dd25e1f0284c8555" 
-      data_hash="4fc300c67452d04e8852cbc315e16010" 
+      data_hash="21a2d025d55b25d6412b1565afb516a5" 
       created_at="1412952242000" updated_at="1412952242000" />
 
 </dataset>

sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml

@@ -1,8 +1,8 @@
 <dataset>
     <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" 
-      data=",,,,,,,,,,,,,unchanged&#13;&#10;,,,,,,,,,,,,,content&#13;&#10;" 
+      data=",,,,,,,,,,,,,,unchanged&#13;&#10;,,,,,,,,,,,,,,content&#13;&#10;" 
       line_hashes="8d7b3d6b83c0a517eac07e1aac94b773&#10;9a0364b9e99bb480dd25e1f0284c8555" 
-      data_hash="4fc300c67452d04e8852cbc315e16010" 
+      data_hash="21a2d025d55b25d6412b1565afb516a5" 
       created_at="1412952242000" updated_at="1412952242000" />
       
     <file_sources id="102" project_uuid="projectUuid" file_uuid="uuidempty" data="[null]"

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/symbol/internal/DefaultSymbol.java

@@ -27,20 +27,14 @@ import java.io.Serializable;
 
 public class DefaultSymbol implements Symbol, org.sonar.api.source.Symbol, Serializable {
 
-  private final String componentKey;
   private final int declarationStartOffset;
   private final int declarationEndOffset;
 
-  public DefaultSymbol(String componentKey, int startOffset, int endOffset) {
-    this.componentKey = componentKey;
+  public DefaultSymbol(int startOffset, int endOffset) {
     this.declarationStartOffset = startOffset;
     this.declarationEndOffset = endOffset;
   }
 
-  public String componentKey() {
-    return componentKey;
-  }
-
   @Override
   public int getDeclarationStartOffset() {
     return declarationStartOffset;
@@ -59,7 +53,6 @@ public class DefaultSymbol implements Symbol, org.sonar.api.source.Symbol, Seria
   @Override
   public String toString() {
     return Objects.toStringHelper("Symbol")
-      .add("component", componentKey)
       .add("offset", String.format("%d-%d", declarationStartOffset, declarationEndOffset))
       .toString();
   }