SONAR-6258 Persist highlighting into file sources

server/sonar-server/src/main/java/org/sonar/server/computation/source/HighlightingLineReader.java

@@ -0,0 +1,158 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.computation.source;
+
+import com.google.common.collect.ImmutableMap;
+import org.sonar.batch.protocol.Constants;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.server.source.db.FileSourceDb;
+
+import javax.annotation.CheckForNull;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.collect.Lists.newArrayList;
+
+public class HighlightingLineReader implements LineReader {
+
+  private static final String OFFSET_SEPARATOR = ",";
+  private static final String ITEM_SEPARATOR = ";";
+
+  private static final Map<Constants.HighlightingType, String> cssClassByType = ImmutableMap.<Constants.HighlightingType, String>builder()
+    .put(Constants.HighlightingType.ANNOTATION, "a")
+    .put(Constants.HighlightingType.CONSTANT, "c")
+    .put(Constants.HighlightingType.COMMENT, "cd")
+    .put(Constants.HighlightingType.STRUCTURED_COMMENT, "j")
+    .put(Constants.HighlightingType.KEYWORD, "k")
+    .put(Constants.HighlightingType.KEYWORD_LIGHT, "h")
+    .put(Constants.HighlightingType.HIGHLIGHTING_STRING, "s")
+    .put(Constants.HighlightingType.PREPROCESS_DIRECTIVE, "p")
+    .build();
+
+  private final Iterator<BatchReport.SyntaxHighlighting> lineHighlightingIterator;
+
+  private BatchReport.SyntaxHighlighting currentItem;
+  private List<BatchReport.SyntaxHighlighting> highlightingList;
+
+  public HighlightingLineReader(Iterator<BatchReport.SyntaxHighlighting> lineHighlightingIterator) {
+    this.lineHighlightingIterator = lineHighlightingIterator;
+    this.highlightingList = newArrayList();
+  }
+
+  @Override
+  public void read(FileSourceDb.Line.Builder lineBuilder) {
+    int line = lineBuilder.getLine();
+    StringBuilder highlighting = new StringBuilder();
+
+    incrementHighlightingListMatchingLine(line);
+    for (Iterator<BatchReport.SyntaxHighlighting> syntaxHighlightingIterator = highlightingList.iterator(); syntaxHighlightingIterator.hasNext(); ) {
+      BatchReport.SyntaxHighlighting syntaxHighlighting = syntaxHighlightingIterator.next();
+      BatchReport.Range range = syntaxHighlighting.getRange();
+      if (range.getStartLine() <= line) {
+        if (highlighting.length() > 0) {
+          highlighting.append(ITEM_SEPARATOR);
+        }
+        highlighting.append(convertHighlightingToString(syntaxHighlighting, line, lineBuilder.getSource()));
+        if (range.getEndLine() == line) {
+          syntaxHighlightingIterator.remove();
+        }
+      }
+    }
+    if (highlighting.length() > 0) {
+      lineBuilder.setHighlighting(highlighting.toString());
+    }
+  }
+
+  private String convertHighlightingToString(BatchReport.SyntaxHighlighting syntaxHighlighting, int line, String sourceLine){
+    BatchReport.Range range = syntaxHighlighting.getRange();
+    validateStartAndEndOffset(range, line);
+
+    StringBuilder symbolLine = new StringBuilder();
+    if (range.getStartLine() == line) {
+      validateStartOffsetNotGreaterThanLineLength(range, sourceLine, line);
+      symbolLine.append(range.getStartOffset()).append(OFFSET_SEPARATOR);
+    } else if (range.getStartLine() < line) {
+      symbolLine.append(0).append(OFFSET_SEPARATOR);
+    }
+
+    if (range.getEndLine() == line) {
+      validateEndOffsetNotGreaterThanLineLength(range, sourceLine, line);
+      symbolLine.append(range.getEndOffset()).append(OFFSET_SEPARATOR);
+    } else if (range.getEndLine() > line) {
+      symbolLine.append(sourceLine.length() - 1).append(OFFSET_SEPARATOR);
+    }
+
+    symbolLine.append(getCssClass(syntaxHighlighting.getType()));
+    return symbolLine.toString();
+  }
+
+  private static String getCssClass(Constants.HighlightingType type) {
+    String cssClass = cssClassByType.get(type);
+    if (cssClass != null) {
+      return cssClass;
+    } else {
+      throw new IllegalArgumentException(String.format("Unknown type %s ", type.toString()));
+    }
+  }
+
+  private void incrementHighlightingListMatchingLine(int line) {
+    BatchReport.SyntaxHighlighting syntaxHighlighting = getNextHighlightingMatchingLine(line);
+    while (syntaxHighlighting != null) {
+      highlightingList.add(syntaxHighlighting);
+      this.currentItem = null;
+      syntaxHighlighting = getNextHighlightingMatchingLine(line);
+    }
+  }
+
+  @CheckForNull
+  private BatchReport.SyntaxHighlighting getNextHighlightingMatchingLine(int line) {
+    // Get next element (if exists)
+    if (currentItem == null && lineHighlightingIterator.hasNext()) {
+      currentItem = lineHighlightingIterator.next();
+    }
+    // Return current element if lines match
+    if (currentItem != null && currentItem.getRange().getStartLine() == line) {
+      return currentItem;
+    }
+    return null;
+  }
+
+  private static void validateStartAndEndOffset(BatchReport.Range range, int line){
+    if (range.getStartLine() == range.getEndLine() && range.getStartOffset() > range.getEndOffset()) {
+      throw new IllegalArgumentException(String.format("End offset %s cannot be defined before start offset %s on line %s", range.getEndOffset(), range.getStartOffset(), line));
+    }
+  }
+
+  private static void validateStartOffsetNotGreaterThanLineLength(BatchReport.Range range, String sourceLine, int line){
+    if (range.getStartOffset() > sourceLine.length()) {
+      throw new IllegalArgumentException(String.format("Start offset %s is defined outside the length (%s) of the line %s", range.getStartOffset(), sourceLine.length(), line));
+    }
+  }
+
+  private static void validateEndOffsetNotGreaterThanLineLength(BatchReport.Range range, String sourceLine, int line){
+    if (range.getEndOffset() > sourceLine.length()) {
+      throw new IllegalArgumentException(String.format("End offset %s is defined outside the length (%s) of the line %s", range.getEndOffset(), sourceLine.length(), line));
+    }
+  }
+
+}

server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java

@@ -20,6 +20,8 @@
 
 package org.sonar.server.computation.step;
 
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.Charsets;
 import org.apache.commons.io.FileUtils;
@@ -113,14 +115,16 @@ public class PersistFileSourcesStep implements ComputationStep {
 
   private List<LineReader> dataLineReaders(BatchReportReader reportReader, int componentRef) {
     List<LineReader> lineReaders = newArrayList();
+
     File coverageFile = reportReader.readComponentCoverage(componentRef);
-    if (coverageFile != null) {
-      lineReaders.add(new CoverageLineReader(new ReportIterator<>(coverageFile, BatchReport.Coverage.PARSER)));
-    }
     BatchReport.Scm scmReport = reportReader.readComponentScm(componentRef);
-    if (scmReport != null) {
-      lineReaders.add(new ScmLineReader(scmReport));
-    }
+    File highlightingFile = reportReader.readComponentSyntaxHighlighting(componentRef);
+
+    lineReaders.add(coverageFile != null ? new CoverageLineReader(new ReportIterator<>(coverageFile, BatchReport.Coverage.PARSER)) : null);
+    lineReaders.add(scmReport != null ? new ScmLineReader(scmReport) : null);
+    lineReaders.add(highlightingFile != null ? new HighlightingLineReader(new ReportIterator<>(highlightingFile, BatchReport.SyntaxHighlighting.PARSER)) : null);
+
+    Iterables.removeIf(lineReaders, Predicates.isNull());
     return lineReaders;
   }
 

server/sonar-server/src/test/java/org/sonar/server/computation/source/HighlightingLineReaderTest.java

@@ -0,0 +1,281 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.computation.source;
+
+import org.junit.Test;
+import org.sonar.batch.protocol.Constants;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.server.source.db.FileSourceDb;
+
+import java.util.Collections;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown;
+
+public class HighlightingLineReaderTest {
+
+  @Test
+  public void nothing_to_read() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(Collections.<BatchReport.SyntaxHighlighting>emptyList().iterator());
+
+    FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1);
+    highlightingLineReader.read(lineBuilder);
+
+    assertThat(lineBuilder.hasHighlighting()).isFalse();
+  }
+
+  @Test
+  public void read_one_line() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(2).setEndOffset(4)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build()
+    ).iterator());
+
+    FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
+    highlightingLineReader.read(lineBuilder);
+
+    assertThat(lineBuilder.getHighlighting()).isEqualTo("2,4,a");
+  }
+
+  @Test
+  public void read_many_lines() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(0).setEndOffset(4)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build(),
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(2).setEndLine(2)
+          .setStartOffset(0).setEndOffset(1)
+          .build())
+        .setType(Constants.HighlightingType.COMMENT)
+        .build(),
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(4).setEndLine(4)
+          .setStartOffset(1).setEndOffset(2)
+          .build())
+        .setType(Constants.HighlightingType.CONSTANT)
+        .build()
+    ).iterator());
+
+    FileSourceDb.Line.Builder line1 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
+    highlightingLineReader.read(line1);
+    FileSourceDb.Line.Builder line2 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line2").setLine(2);
+    highlightingLineReader.read(line2);
+    highlightingLineReader.read(FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line3").setLine(3));
+    FileSourceDb.Line.Builder line4 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line4").setLine(4);
+    highlightingLineReader.read(line4);
+
+    assertThat(line1.getHighlighting()).isEqualTo("0,4,a");
+    assertThat(line2.getHighlighting()).isEqualTo("0,1,cd");
+    assertThat(line4.getHighlighting()).isEqualTo("1,2,c");
+  }
+
+  @Test
+  public void read_many_syntax_highlighting_on_same_line() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(2).setEndOffset(3)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build(),
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(4).setEndOffset(5)
+          .build())
+        .setType(Constants.HighlightingType.COMMENT)
+        .build()
+    ).iterator());
+
+    FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
+    highlightingLineReader.read(lineBuilder);
+
+    assertThat(lineBuilder.getHighlighting()).isEqualTo("2,3,a;4,5,cd");
+  }
+
+  @Test
+  public void read_nested_syntax_highlighting_on_same_line() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(0).setEndOffset(4)
+          .build())
+        .setType(Constants.HighlightingType.CONSTANT)
+        .build(),
+      // This highlighting is nested in previous one
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(2).setEndOffset(3)
+          .build())
+        .setType(Constants.HighlightingType.KEYWORD)
+        .build()
+    ).iterator());
+
+    FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
+    highlightingLineReader.read(lineBuilder);
+
+    assertThat(lineBuilder.getHighlighting()).isEqualTo("0,4,c;2,3,k");
+  }
+
+  @Test
+  public void read_one_syntax_highlighting_on_many_lines() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      // This highlighting begin on line 1 and finish on line 3
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(3)
+          .setStartOffset(3).setEndOffset(2)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build()
+    ).iterator());
+
+    FileSourceDb.Line.Builder line1 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
+    highlightingLineReader.read(line1);
+    FileSourceDb.Line.Builder line2 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line 2").setLine(2);
+    highlightingLineReader.read(line2);
+    FileSourceDb.Line.Builder line3 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line3").setLine(3);
+    highlightingLineReader.read(line3);
+
+    assertThat(line1.getHighlighting()).isEqualTo("3,4,a");
+    assertThat(line2.getHighlighting()).isEqualTo("0,5,a");
+    assertThat(line3.getHighlighting()).isEqualTo("0,2,a");
+  }
+
+  // TODO
+  @Test
+  public void read_many_syntax_highlighting_on_many_lines() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(3)
+          .setStartOffset(3).setEndOffset(2)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build(),
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(2).setEndLine(4)
+          .setStartOffset(0).setEndOffset(3)
+          .build())
+        .setType(Constants.HighlightingType.HIGHLIGHTING_STRING)
+        .build(),
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(2).setEndLine(2)
+          .setStartOffset(1).setEndOffset(2)
+          .build())
+        .setType(Constants.HighlightingType.COMMENT)
+        .build()
+    ).iterator());
+
+    FileSourceDb.Line.Builder line1 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1);
+    highlightingLineReader.read(line1);
+    FileSourceDb.Line.Builder line2 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line2").setLine(2);
+    highlightingLineReader.read(line2);
+    FileSourceDb.Line.Builder line3 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line3").setLine(3);
+    highlightingLineReader.read(line3);
+    FileSourceDb.Line.Builder line4 = FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line4").setLine(4);
+    highlightingLineReader.read(line4);
+
+    assertThat(line1.getHighlighting()).isEqualTo("3,4,a");
+    assertThat(line2.getHighlighting()).isEqualTo("0,4,a;0,4,s;1,2,cd");
+    assertThat(line3.getHighlighting()).isEqualTo("0,2,a;0,4,s");
+    assertThat(line4.getHighlighting()).isEqualTo("0,3,s");
+  }
+
+  @Test
+  public void fail_when_end_offset_is_before_start_offset() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(4).setEndOffset(2)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build()
+    ).iterator());
+
+    try {
+      highlightingLineReader.read(FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1));
+      failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
+    } catch (IllegalArgumentException e) {
+      assertThat(e).hasMessage("End offset 2 cannot be defined before start offset 4 on line 1");
+    }
+  }
+
+  @Test
+  public void fail_when_end_offset_is_higher_than_line_length() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(2).setEndOffset(10)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build()
+    ).iterator());
+
+    try {
+      highlightingLineReader.read(FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1));
+      failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
+    } catch (IllegalArgumentException e) {
+      assertThat(e).hasMessage("End offset 10 is defined outside the length (5) of the line 1");
+    }
+  }
+
+  @Test
+  public void fail_when_start_offset_is_higher_than_line_length() {
+    HighlightingLineReader highlightingLineReader = new HighlightingLineReader(newArrayList(
+      BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(10).setEndOffset(11)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build()
+    ).iterator());
+
+    try {
+      highlightingLineReader.read(FileSourceDb.Data.newBuilder().addLinesBuilder().setSource("line1").setLine(1));
+      failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
+    } catch (IllegalArgumentException e) {
+      assertThat(e).hasMessage("Start offset 10 is defined outside the length (5) of the line 1");
+    }
+  }
+
+}

server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java

@@ -43,6 +43,7 @@ import org.sonar.test.DbTests;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.List;
 
 import static com.google.common.collect.Lists.newArrayList;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -100,16 +101,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
 
   @Test
   public void persist_sources() throws Exception {
-    BatchReportWriter writer = initReport();
-    writer.writeComponent(BatchReport.Component.newBuilder()
-      .setRef(FILE_REF)
-      .setType(Constants.ComponentType.FILE)
-      .setUuid(FILE_UUID)
-      .setLines(2)
-      .build());
-
-    File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF);
-    FileUtils.writeLines(sourceFile, Lists.newArrayList("line1", "line2"));
+    initBasicReport(2);
 
     sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID)));
 
@@ -133,7 +125,18 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
 
   @Test
   public void persist_last_line() throws Exception {
-    BatchReportWriter writer = initReport();
+    BatchReportWriter writer = new BatchReportWriter(reportDir);
+    FileUtils.writeLines(writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF), Lists.newArrayList("line1", "line2"));
+    writer.writeMetadata(BatchReport.Metadata.newBuilder()
+      .setRootComponentRef(1)
+      .setProjectKey("PROJECT_KEY")
+      .build());
+    writer.writeComponent(BatchReport.Component.newBuilder()
+      .setRef(1)
+      .setType(Constants.ComponentType.PROJECT)
+      .setUuid(PROJECT_UUID)
+      .addChildRef(FILE_REF)
+      .build());
     writer.writeComponent(BatchReport.Component.newBuilder()
       .setRef(FILE_REF)
       .setType(Constants.ComponentType.FILE)
@@ -142,9 +145,6 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
       .setLines(3)
       .build());
 
-    File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF);
-    FileUtils.writeLines(sourceFile, Lists.newArrayList("line1", "line2"));
-
     sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID)));
 
     assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1);
@@ -157,34 +157,19 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
 
   @Test
   public void persist_source_hashes() throws Exception {
-    BatchReportWriter writer = initReport();
-    writer.writeComponent(BatchReport.Component.newBuilder()
-      .setRef(FILE_REF)
-      .setType(Constants.ComponentType.FILE)
-      .setUuid(FILE_UUID)
-      .setLines(2)
-      .build());
-
-    File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF);
-    FileUtils.writeLines(sourceFile, Lists.newArrayList("line\t1", "line2"));
+    initBasicReport(2);
 
     sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID)));
 
     assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1);
     FileSourceDto fileSourceDto = dbClient.fileSourceDao().select("FILE");
     assertThat(fileSourceDto.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\ne6251bcf1a7dc3ba5e7933e325bbe605\n");
-    assertThat(fileSourceDto.getSrcHash()).isEqualTo("fe1ac2747e8393015698f2724d8d2835");
+    assertThat(fileSourceDto.getSrcHash()).isEqualTo("4fcc82a88ee38e0aa16c17f512c685c9");
   }
 
   @Test
   public void persist_coverage() throws Exception {
-    BatchReportWriter writer = initReport();
-    writer.writeComponent(BatchReport.Component.newBuilder()
-      .setRef(FILE_REF)
-      .setType(Constants.ComponentType.FILE)
-      .setUuid(FILE_UUID)
-      .setLines(1)
-      .build());
+    BatchReportWriter writer = initBasicReport(1);
 
     writer.writeComponentCoverage(FILE_REF, newArrayList(BatchReport.Coverage.newBuilder()
       .setLine(1)
@@ -196,9 +181,6 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
       .setOverallCoveredConditions(4)
       .build()));
 
-    File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF);
-    FileUtils.writeLines(sourceFile, Lists.newArrayList("line1"));
-
     sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID)));
 
     assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1);
@@ -220,13 +202,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
 
   @Test
   public void persist_scm() throws Exception {
-    BatchReportWriter writer = initReport();
-    writer.writeComponent(BatchReport.Component.newBuilder()
-      .setRef(FILE_REF)
-      .setType(Constants.ComponentType.FILE)
-      .setUuid(FILE_UUID)
-      .setLines(1)
-      .build());
+    BatchReportWriter writer = initBasicReport(1);
 
     writer.writeComponentScm(BatchReport.Scm.newBuilder()
         .setComponentRef(FILE_REF)
@@ -239,9 +215,6 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
         .build()
     );
 
-    File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF);
-    FileUtils.writeLines(sourceFile, Lists.newArrayList("line1"));
-
     sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID)));
 
     assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1);
@@ -255,13 +228,37 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
     assertThat(data.getLines(0).getScmRevision()).isEqualTo("rev-1");
   }
 
+  @Test
+  public void persist_highlighting() throws Exception {
+    BatchReportWriter writer = initBasicReport(1);
+
+    writer.writeComponentSyntaxHighlighting(FILE_REF, newArrayList(BatchReport.SyntaxHighlighting.newBuilder()
+        .setRange(BatchReport.Range.newBuilder()
+          .setStartLine(1).setEndLine(1)
+          .setStartOffset(2).setEndOffset(4)
+          .build())
+        .setType(Constants.HighlightingType.ANNOTATION)
+        .build())
+    );
+
+    sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID)));
+
+    assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1);
+    FileSourceDto fileSourceDto = dbClient.fileSourceDao().select(FILE_UUID);
+    FileSourceDb.Data data = FileSourceDto.decodeData(fileSourceDto.getBinaryData());
+
+    assertThat(data.getLinesList()).hasSize(1);
+
+    assertThat(data.getLines(0).getHighlighting()).isEqualTo("2,4,a");
+  }
+
   @Test
   public void not_update_sources_when_nothing_has_changed() throws Exception {
     // Existing sources
     long past = 150000L;
-    String srcHash = "5b4bd9815cdb17b8ceae19eb1810c34c";
-    String lineHashes = "6438c669e0d0de98e6929c2cc0fac474\n";
-    String dataHash = "6cad150e3d065976c230cddc5a09efaa";
+    String srcHash = "1ddab9058a07abc0db2605ab02a61a00";
+    String lineHashes = "137f72c3708c6bd0de00a0e5a69c699b\n";
+    String dataHash = "29f25900140c94db38035128cb6de6a2";
 
     dbClient.fileSourceDao().insert(session, new FileSourceDto()
       .setProjectUuid(PROJECT_UUID)
@@ -272,7 +269,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
       .setBinaryData(FileSourceDto.encodeData(FileSourceDb.Data.newBuilder()
         .addLines(FileSourceDb.Line.newBuilder()
           .setLine(1)
-          .setSource("line")
+          .setSource("line1")
           .build())
         .build()))
       .setCreatedAt(past)
@@ -280,16 +277,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
     session.commit();
 
     // Sources from the report
-    BatchReportWriter writer = initReport();
-    writer.writeComponent(BatchReport.Component.newBuilder()
-      .setRef(FILE_REF)
-      .setType(Constants.ComponentType.FILE)
-      .setUuid(FILE_UUID)
-      .setLines(1)
-      .build());
-
-    File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF);
-    FileUtils.writeLines(sourceFile, Lists.newArrayList("line"));
+    initBasicReport(1);
 
     sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID)));
 
@@ -315,24 +303,14 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
       .setBinaryData(FileSourceDto.encodeData(FileSourceDb.Data.newBuilder()
         .addLines(FileSourceDb.Line.newBuilder()
           .setLine(1)
-          .setSource("line")
+          .setSource("old line")
           .build())
         .build()))
       .setCreatedAt(past)
       .setUpdatedAt(past));
     session.commit();
 
-    // New sources from the report
-    BatchReportWriter writer = initReport();
-    writer.writeComponent(BatchReport.Component.newBuilder()
-      .setRef(FILE_REF)
-      .setType(Constants.ComponentType.FILE)
-      .setUuid(FILE_UUID)
-      .setLines(1)
-      .build());
-
-    File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF);
-    FileUtils.writeLines(sourceFile, Lists.newArrayList("new line"));
+    initBasicReport(1);
 
     sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID)));
 
@@ -349,9 +327,9 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
     dbClient.fileSourceDao().insert(session, new FileSourceDto()
       .setProjectUuid(PROJECT_UUID)
       .setFileUuid(FILE_UUID)
-      // Source hash is missing, udate will be made
-      .setLineHashes("6438c669e0d0de98e6929c2cc0fac474\n")
-      .setDataHash("6cad150e3d065976c230cddc5a09efaa")
+      // Source hash is missing, update will be made
+      .setLineHashes("137f72c3708c6bd0de00a0e5a69c699b\n")
+      .setDataHash("29f25900140c94db38035128cb6de6a2")
       .setBinaryData(FileSourceDto.encodeData(FileSourceDb.Data.newBuilder()
         .addLines(FileSourceDb.Line.newBuilder()
           .setLine(1)
@@ -362,17 +340,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
       .setUpdatedAt(past));
     session.commit();
 
-    // New sources from the report
-    BatchReportWriter writer = initReport();
-    writer.writeComponent(BatchReport.Component.newBuilder()
-      .setRef(FILE_REF)
-      .setType(Constants.ComponentType.FILE)
-      .setUuid(FILE_UUID)
-      .setLines(1)
-      .build());
-
-    File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF);
-    FileUtils.writeLines(sourceFile, Lists.newArrayList("line"));
+    initBasicReport(1);
 
     sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID)));
 
@@ -381,10 +349,10 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
     assertThat(fileSourceDto.getCreatedAt()).isEqualTo(past);
     // Updated at is not updated to not reindex the file source in E/S as the src hash is not indexed
     assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(past);
-    assertThat(fileSourceDto.getSrcHash()).isEqualTo("5b4bd9815cdb17b8ceae19eb1810c34c");
+    assertThat(fileSourceDto.getSrcHash()).isEqualTo("1ddab9058a07abc0db2605ab02a61a00");
   }
 
-  private BatchReportWriter initReport() throws IOException {
+  private BatchReportWriter initBasicReport(int numberOfLines) throws IOException {
     BatchReportWriter writer = new BatchReportWriter(reportDir);
     writer.writeMetadata(BatchReport.Metadata.newBuilder()
       .setRootComponentRef(1)
@@ -403,6 +371,18 @@ public class PersistFileSourcesStepTest extends BaseStepTest {
       .setUuid("MODULE")
       .addChildRef(FILE_REF)
       .build());
+    writer.writeComponent(BatchReport.Component.newBuilder()
+      .setRef(FILE_REF)
+      .setType(Constants.ComponentType.FILE)
+      .setUuid(FILE_UUID)
+      .setLines(numberOfLines)
+      .build());
+
+    List<String> lines = newArrayList();
+    for (int i=1; i<=numberOfLines; i++) {
+      lines.add("line" + i);
+    }
+    FileUtils.writeLines(writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF), lines);
 
     return writer;
   }