Merge pull request #191 from SonarSource/feature/batch/feed_coverage_report

SONAR-6339 Feed Coverage in compute report

14 files changed, 541 insertions, 83 deletions

diff --git a/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/output/BatchReportWriter.java b/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/output/BatchReportWriter.java
index 97b7e5f2caa..86ec1c53f6e 100644
--- a/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/output/BatchReportWriter.java
+++ b/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/output/BatchReportWriter.java
@@ -118,4 +118,8 @@ public class BatchReportWriter {
     ProtobufUtil.writeMessagesToFile(coverageList, file);
   }
 
+  public File getSourceFile(int componentRef) {
+    return fileStructure.fileFor(FileStructure.Domain.SOURCE, componentRef);
+  }
+
 }
diff --git a/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/output/FileStructure.java b/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/output/FileStructure.java
index 21df5a4cafb..e0add6865a0 100644
--- a/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/output/FileStructure.java
+++ b/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/output/FileStructure.java
@@ -27,22 +27,24 @@ import java.io.File;
 public class FileStructure {
 
   public enum Domain {
-    ISSUES("issues-"),
-    ISSUES_ON_DELETED("issues-deleted-"),
-    COMPONENT("component-"),
-    MEASURES("measures-"),
-    DUPLICATIONS("duplications-"),
-    SYNTAX_HIGHLIGHTING("syntax-highlighting-"),
-    SCM("scm-"),
-    SYMBOLS("symbol-"),
-    COVERAGE("coverage-"),
-    SOURCE("source-")
-    ;
-    
+    ISSUES("issues-", Domain.PB),
+    ISSUES_ON_DELETED("issues-deleted-", Domain.PB),
+    COMPONENT("component-", Domain.PB),
+    MEASURES("measures-", Domain.PB),
+    DUPLICATIONS("duplications-", Domain.PB),
+    SYNTAX_HIGHLIGHTING("syntax-highlighting-", Domain.PB),
+    SCM("scm-", Domain.PB),
+    SYMBOLS("symbol-", Domain.PB),
+    COVERAGE("coverage-", Domain.PB),
+    SOURCE("source-", ".txt");
+
+    private static final String PB = ".pb";
     private final String filePrefix;
+    private final String fileSuffix;
 
-    Domain(String filePrefix) {
+    Domain(String filePrefix, String fileSuffix) {
       this.filePrefix = filePrefix;
+      this.fileSuffix = fileSuffix;
     }
   }
 
@@ -60,7 +62,7 @@ public class FileStructure {
   }
 
   public File fileFor(Domain domain, int componentRef) {
-    return new File(dir, domain.filePrefix + componentRef + ".pb");
+    return new File(dir, domain.filePrefix + componentRef + domain.fileSuffix);
   }
 
 }
diff --git a/sonar-batch-protocol/src/test/java/org/sonar/batch/protocol/output/BatchReportReaderTest.java b/sonar-batch-protocol/src/test/java/org/sonar/batch/protocol/output/BatchReportReaderTest.java
index c109796fd35..184e1209804 100644
--- a/sonar-batch-protocol/src/test/java/org/sonar/batch/protocol/output/BatchReportReaderTest.java
+++ b/sonar-batch-protocol/src/test/java/org/sonar/batch/protocol/output/BatchReportReaderTest.java
@@ -196,8 +196,7 @@ public class BatchReportReaderTest {
 
     sut = new BatchReportReader(dir);
 
-    InputStream inputStream = FileUtils.openInputStream(new BatchReportReader(dir).readFileCoverage(1));
-    try {
+    try (InputStream inputStream = FileUtils.openInputStream(new BatchReportReader(dir).readFileCoverage(1))) {
       BatchReport.Coverage coverage = BatchReport.Coverage.PARSER.parseDelimitedFrom(inputStream);
       assertThat(coverage.getLine()).isEqualTo(1);
       assertThat(coverage.getConditions()).isEqualTo(1);
@@ -215,8 +214,6 @@ public class BatchReportReaderTest {
       assertThat(coverage.getUtCoveredConditions()).isEqualTo(4);
       assertThat(coverage.getItCoveredConditions()).isEqualTo(5);
       assertThat(coverage.getOverallCoveredConditions()).isEqualTo(5);
-    } finally {
-      inputStream.close();
     }
   }
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/report/CoveragePublisher.java b/sonar-batch/src/main/java/org/sonar/batch/report/CoveragePublisher.java
new file mode 100644
index 00000000000..6bc33a24c6e
--- /dev/null
+++ b/sonar-batch/src/main/java/org/sonar/batch/report/CoveragePublisher.java
@@ -0,0 +1,130 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.report;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import org.apache.commons.lang.StringUtils;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.measures.CoreMetrics;
+import org.sonar.api.measures.Measure;
+import org.sonar.api.utils.KeyValueFormat;
+import org.sonar.batch.index.BatchResource;
+import org.sonar.batch.index.ResourceCache;
+import org.sonar.batch.protocol.output.BatchReport.Coverage;
+import org.sonar.batch.protocol.output.BatchReport.Coverage.Builder;
+import org.sonar.batch.protocol.output.BatchReportWriter;
+import org.sonar.batch.scan.measure.MeasureCache;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class CoveragePublisher implements ReportPublisherStep {
+
+  private final ResourceCache resourceCache;
+  private final MeasureCache measureCache;
+
+  public CoveragePublisher(ResourceCache resourceCache, MeasureCache measureCache) {
+    this.resourceCache = resourceCache;
+    this.measureCache = measureCache;
+  }
+
+  @Override
+  public void publish(BatchReportWriter writer) {
+    for (final BatchResource resource : resourceCache.all()) {
+      if (!resource.isFile()) {
+        continue;
+      }
+      Map<Integer, Coverage.Builder> coveragePerLine = new LinkedHashMap<>();
+
+      applyLineMeasure(resource.key(), ((InputFile) resource.inputPath()).lines(), CoreMetrics.COVERAGE_LINE_HITS_DATA_KEY, coveragePerLine, new MeasureOperation() {
+        @Override
+        public void apply(String value, Coverage.Builder builder) {
+          builder.setUtHits(Integer.parseInt(value) > 0);
+        }
+      });
+      applyLineMeasure(resource.key(), ((InputFile) resource.inputPath()).lines(), CoreMetrics.CONDITIONS_BY_LINE_KEY, coveragePerLine, new MeasureOperation() {
+        @Override
+        public void apply(String value, Coverage.Builder builder) {
+          builder.setConditions(Integer.parseInt(value));
+        }
+      });
+      applyLineMeasure(resource.key(), ((InputFile) resource.inputPath()).lines(), CoreMetrics.COVERED_CONDITIONS_BY_LINE_KEY, coveragePerLine, new MeasureOperation() {
+        @Override
+        public void apply(String value, Coverage.Builder builder) {
+          builder.setUtCoveredConditions(Integer.parseInt(value));
+        }
+      });
+      applyLineMeasure(resource.key(), ((InputFile) resource.inputPath()).lines(), CoreMetrics.IT_COVERAGE_LINE_HITS_DATA_KEY, coveragePerLine, new MeasureOperation() {
+        @Override
+        public void apply(String value, Coverage.Builder builder) {
+          builder.setItHits(Integer.parseInt(value) > 0);
+        }
+      });
+      applyLineMeasure(resource.key(), ((InputFile) resource.inputPath()).lines(), CoreMetrics.IT_COVERED_CONDITIONS_BY_LINE_KEY, coveragePerLine, new MeasureOperation() {
+        @Override
+        public void apply(String value, Coverage.Builder builder) {
+          builder.setItCoveredConditions(Integer.parseInt(value));
+        }
+      });
+      applyLineMeasure(resource.key(), ((InputFile) resource.inputPath()).lines(), CoreMetrics.OVERALL_COVERED_CONDITIONS_BY_LINE_KEY, coveragePerLine, new MeasureOperation() {
+        @Override
+        public void apply(String value, Coverage.Builder builder) {
+          builder.setOverallCoveredConditions(Integer.parseInt(value));
+        }
+      });
+
+      writer.writeFileCoverage(resource.batchId(), Iterables.transform(coveragePerLine.values(), new Function<Coverage.Builder, Coverage>() {
+        @Override
+        public Coverage apply(Builder input) {
+          return input.build();
+        }
+      }));
+    }
+  }
+
+  void applyLineMeasure(String inputFileKey, int lineCount, String metricKey, Map<Integer, Coverage.Builder> coveragePerLine, MeasureOperation op) {
+    Iterable<Measure> measures = measureCache.byMetric(inputFileKey, metricKey);
+    if (measures.iterator().hasNext()) {
+      Measure measure = measures.iterator().next();
+      Map<Integer, String> lineMeasures = KeyValueFormat.parseIntString((String) measure.value());
+      for (Map.Entry<Integer, String> lineMeasure : lineMeasures.entrySet()) {
+        int lineIdx = lineMeasure.getKey();
+        if (lineIdx <= lineCount) {
+          String value = lineMeasure.getValue();
+          if (StringUtils.isNotEmpty(value)) {
+            Coverage.Builder coverageBuilder = coveragePerLine.get(lineIdx);
+            if (coverageBuilder == null) {
+              coverageBuilder = Coverage.newBuilder();
+              coverageBuilder.setLine(lineIdx);
+              coveragePerLine.put(lineIdx, coverageBuilder);
+            }
+            op.apply(value, coverageBuilder);
+          }
+        }
+      }
+    }
+  }
+
+  static interface MeasureOperation {
+    void apply(String value, Coverage.Builder builder);
+  }
+
+}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/report/DuplicationsPublisher.java b/sonar-batch/src/main/java/org/sonar/batch/report/DuplicationsPublisher.java
index 0295e13bb2b..9b0dde87911 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/report/DuplicationsPublisher.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/report/DuplicationsPublisher.java
@@ -44,6 +44,9 @@ public class DuplicationsPublisher implements ReportPublisherStep {
   @Override
   public void publish(BatchReportWriter writer) {
     for (final BatchResource resource : resourceCache.all()) {
+      if (!resource.isFile()) {
+        continue;
+      }
       Iterable<DefaultDuplication> dups = duplicationCache.byComponent(resource.resource().getEffectiveKey());
       if (dups.iterator().hasNext()) {
         Iterable<org.sonar.batch.protocol.output.BatchReport.Duplication> reportDuplications = Iterables.transform(dups,
diff --git a/sonar-batch/src/main/java/org/sonar/batch/report/SourcePublisher.java b/sonar-batch/src/main/java/org/sonar/batch/report/SourcePublisher.java
new file mode 100644
index 00000000000..8f6b7f3ffb5
--- /dev/null
+++ b/sonar-batch/src/main/java/org/sonar/batch/report/SourcePublisher.java
@@ -0,0 +1,68 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.report;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.input.BOMInputStream;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.batch.index.BatchResource;
+import org.sonar.batch.index.ResourceCache;
+import org.sonar.batch.protocol.output.BatchReportWriter;
+
+import java.io.*;
+
+public class SourcePublisher implements ReportPublisherStep {
+
+  private final ResourceCache resourceCache;
+
+  public SourcePublisher(ResourceCache resourceCache) {
+    this.resourceCache = resourceCache;
+  }
+
+  @Override
+  public void publish(BatchReportWriter writer) {
+    for (final BatchResource resource : resourceCache.all()) {
+      if (!resource.isFile()) {
+        continue;
+      }
+
+      DefaultInputFile inputFile = (DefaultInputFile) resource.inputPath();
+      File iofile = writer.getSourceFile(resource.batchId());
+      int line = 0;
+      try (FileOutputStream output = new FileOutputStream(iofile); BOMInputStream bomIn = new BOMInputStream(new FileInputStream(inputFile.file()),
+        ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE);
+        BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, inputFile.charset()))) {
+        String lineStr = reader.readLine();
+        while (lineStr != null) {
+          IOUtils.write(lineStr, output, Charsets.UTF_8);
+          line++;
+          if (line < inputFile.lines()) {
+            IOUtils.write("\n", output, Charsets.UTF_8);
+          }
+          lineStr = reader.readLine();
+        }
+      } catch (IOException e) {
+        throw new IllegalStateException("Unable to store file source in the report", e);
+      }
+    }
+  }
+}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java
index 6e724866c67..d9a67454ad0 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java
@@ -57,7 +57,6 @@ import org.sonar.batch.language.LanguageDistributionDecorator;
 import org.sonar.batch.phases.*;
 import org.sonar.batch.qualitygate.GenerateQualityGateEvents;
 import org.sonar.batch.qualitygate.QualityGateVerifier;
-import org.sonar.batch.report.*;
 import org.sonar.batch.rule.*;
 import org.sonar.batch.scan.filesystem.*;
 import org.sonar.batch.scan.report.IssuesReports;
@@ -114,11 +113,6 @@ public class ModuleScanContainer extends ComponentContainer {
       SensorsExecutor.class,
       InitializersExecutor.class,
       ProjectInitializer.class,
-      ReportPublisher.class,
-      ComponentsPublisher.class,
-      IssuesPublisher.class,
-      MeasuresPublisher.class,
-      DuplicationsPublisher.class,
       moduleDefinition.getContainerExtensions(),
 
       // file system
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
index 30bfafa9793..12b0bc0f21d 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
@@ -186,6 +186,9 @@ public class ProjectScanContainer extends ComponentContainer {
       ComponentsPublisher.class,
       IssuesPublisher.class,
       MeasuresPublisher.class,
+      DuplicationsPublisher.class,
+      CoveragePublisher.class,
+      SourcePublisher.class,
 
       ScanTaskObservers.class);
   }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java b/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java
index 724c900176e..5aec595a9f7 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java
@@ -22,8 +22,10 @@ package org.sonar.batch.sensor;
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
-import org.sonar.api.batch.fs.*;
+import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.InputPath;
+import org.sonar.api.batch.fs.TextRange;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.measure.MetricFinder;
 import org.sonar.api.batch.rule.ActiveRules;
@@ -43,7 +45,10 @@ import org.sonar.api.measures.Formula;
 import org.sonar.api.measures.Metric;
 import org.sonar.api.measures.PersistenceMode;
 import org.sonar.api.measures.SumChildDistributionFormula;
-import org.sonar.api.resources.*;
+import org.sonar.api.resources.Directory;
+import org.sonar.api.resources.File;
+import org.sonar.api.resources.Project;
+import org.sonar.api.resources.Scopes;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.source.Symbol;
 import org.sonar.batch.duplication.DuplicationCache;
@@ -151,18 +156,14 @@ public class DefaultSensorStorage implements SensorStorage {
 
   @Override
   public void store(Issue issue) {
-    Resource r;
+    String componentKey;
     InputPath inputPath = issue.inputPath();
     if (inputPath != null) {
-      if (inputPath instanceof InputDir) {
-        r = Directory.create(inputPath.relativePath());
-      } else {
-        r = File.create(inputPath.relativePath());
-      }
+      componentKey = ComponentKeys.createEffectiveKey(project.getKey(), inputPath);
     } else {
-      r = project;
+      componentKey = project.getKey();
     }
-    moduleIssues.initAndAddIssue(toDefaultIssue(project.getKey(), ComponentKeys.createEffectiveKey(project, r), issue));
+    moduleIssues.initAndAddIssue(toDefaultIssue(project.getKey(), componentKey, issue));
   }
 
   public static DefaultIssue toDefaultIssue(String projectKey, String componentKey, Issue issue) {
@@ -178,33 +179,12 @@ public class DefaultSensorStorage implements SensorStorage {
       .build();
   }
 
-  private File getTestResource(InputFile testFile) {
-    File testRes = File.create(testFile.relativePath());
-    testRes.setQualifier(Qualifiers.UNIT_TEST_FILE);
-    // Reload
-    testRes = sonarIndex.getResource(testRes);
-    if (testRes == null) {
-      throw new IllegalArgumentException("Provided input file is not indexed or not a test file: " + testFile);
-    }
-    return testRes;
-  }
-
-  private File getMainResource(InputFile mainFile) {
-    File mainRes = File.create(mainFile.relativePath());
-    // Reload
-    mainRes = sonarIndex.getResource(mainRes);
-    if (mainRes == null) {
-      throw new IllegalArgumentException("Provided input file is not indexed or not a main file: " + mainRes);
-    }
-    return mainRes;
-  }
-
   private File getFile(InputFile file) {
-    if (file.type() == InputFile.Type.MAIN) {
-      return getMainResource(file);
-    } else {
-      return getTestResource(file);
+    BatchResource r = resourceCache.get(file);
+    if (r == null) {
+      throw new IllegalStateException("Provided input file is not indexed");
     }
+    return (File) r.resource();
   }
 
   @Override
diff --git a/sonar-batch/src/test/java/org/sonar/batch/report/CoveragePublisherTest.java b/sonar-batch/src/test/java/org/sonar/batch/report/CoveragePublisherTest.java
new file mode 100644
index 00000000000..3e3c4cf284d
--- /dev/null
+++ b/sonar-batch/src/test/java/org/sonar/batch/report/CoveragePublisherTest.java
@@ -0,0 +1,122 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.report;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.database.model.Snapshot;
+import org.sonar.api.measures.CoreMetrics;
+import org.sonar.api.measures.Measure;
+import org.sonar.api.resources.Project;
+import org.sonar.batch.index.ResourceCache;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.batch.protocol.output.BatchReport.Coverage;
+import org.sonar.batch.protocol.output.BatchReportReader;
+import org.sonar.batch.protocol.output.BatchReportWriter;
+import org.sonar.batch.scan.measure.MeasureCache;
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CoveragePublisherTest {
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  private MeasureCache measureCache;
+  private CoveragePublisher publisher;
+
+  private org.sonar.api.resources.Resource sampleFile;
+
+  @Before
+  public void prepare() {
+    Project p = new Project("foo").setAnalysisDate(new Date(1234567L));
+    ResourceCache resourceCache = new ResourceCache();
+    sampleFile = org.sonar.api.resources.File.create("src/Foo.php").setEffectiveKey("foo:src/Foo.php");
+    resourceCache.add(p, null).setSnapshot(new Snapshot().setId(2));
+    resourceCache.add(sampleFile, null).setInputPath(new DefaultInputFile("foo", "src/Foo.php").setLines(5));
+    measureCache = mock(MeasureCache.class);
+    when(measureCache.byMetric(anyString(), anyString())).thenReturn(Collections.<Measure>emptyList());
+    publisher = new CoveragePublisher(resourceCache, measureCache);
+  }
+
+  @Test
+  public void publishCoverage() throws Exception {
+
+    Measure utLineHits = new Measure<>(CoreMetrics.COVERAGE_LINE_HITS_DATA).setData("2=1;3=1;5=0;6=3");
+    when(measureCache.byMetric("foo:src/Foo.php", CoreMetrics.COVERAGE_LINE_HITS_DATA_KEY)).thenReturn(Arrays.asList(utLineHits));
+
+    Measure conditionsByLine = new Measure<>(CoreMetrics.CONDITIONS_BY_LINE).setData("3=4");
+    when(measureCache.byMetric("foo:src/Foo.php", CoreMetrics.CONDITIONS_BY_LINE_KEY)).thenReturn(Arrays.asList(conditionsByLine));
+
+    Measure coveredConditionsByUts = new Measure<>(CoreMetrics.COVERED_CONDITIONS_BY_LINE).setData("3=2");
+    when(measureCache.byMetric("foo:src/Foo.php", CoreMetrics.COVERED_CONDITIONS_BY_LINE_KEY)).thenReturn(Arrays.asList(coveredConditionsByUts));
+
+    Measure itLineHits = new Measure<>(CoreMetrics.IT_COVERAGE_LINE_HITS_DATA).setData("2=0;3=0;5=1");
+    when(measureCache.byMetric("foo:src/Foo.php", CoreMetrics.IT_COVERAGE_LINE_HITS_DATA_KEY)).thenReturn(Arrays.asList(itLineHits));
+
+    Measure coveredConditionsByIts = new Measure<>(CoreMetrics.IT_COVERED_CONDITIONS_BY_LINE).setData("3=1");
+    when(measureCache.byMetric("foo:src/Foo.php", CoreMetrics.IT_COVERED_CONDITIONS_BY_LINE_KEY)).thenReturn(Arrays.asList(coveredConditionsByIts));
+
+    Measure overallCoveredConditions = new Measure<>(CoreMetrics.OVERALL_COVERED_CONDITIONS_BY_LINE).setData("3=2");
+    when(measureCache.byMetric("foo:src/Foo.php", CoreMetrics.OVERALL_COVERED_CONDITIONS_BY_LINE_KEY)).thenReturn(Arrays.asList(overallCoveredConditions));
+
+    File outputDir = temp.newFolder();
+    BatchReportWriter writer = new BatchReportWriter(outputDir);
+
+    publisher.publish(writer);
+
+    try (InputStream inputStream = FileUtils.openInputStream(new BatchReportReader(outputDir).readFileCoverage(2))) {
+      assertThat(BatchReport.Coverage.PARSER.parseDelimitedFrom(inputStream)).isEqualTo(Coverage.newBuilder()
+        .setLine(2)
+        .setUtHits(true)
+        .setItHits(false)
+        .build());
+      assertThat(BatchReport.Coverage.PARSER.parseDelimitedFrom(inputStream)).isEqualTo(Coverage.newBuilder()
+        .setLine(3)
+        .setUtHits(true)
+        .setItHits(false)
+        .setConditions(4)
+        .setUtCoveredConditions(2)
+        .setItCoveredConditions(1)
+        .setOverallCoveredConditions(2)
+        .build());
+      assertThat(BatchReport.Coverage.PARSER.parseDelimitedFrom(inputStream)).isEqualTo(Coverage.newBuilder()
+        .setLine(5)
+        .setUtHits(false)
+        .setItHits(true)
+        .build());
+
+    }
+  }
+
+}
diff --git a/sonar-batch/src/test/java/org/sonar/batch/report/SourcePublisherTest.java b/sonar-batch/src/test/java/org/sonar/batch/report/SourcePublisherTest.java
new file mode 100644
index 00000000000..e9b379a58e2
--- /dev/null
+++ b/sonar-batch/src/test/java/org/sonar/batch/report/SourcePublisherTest.java
@@ -0,0 +1,104 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.report;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.database.model.Snapshot;
+import org.sonar.api.resources.Project;
+import org.sonar.batch.index.ResourceCache;
+import org.sonar.batch.protocol.output.BatchReportWriter;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SourcePublisherTest {
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  private SourcePublisher publisher;
+
+  private File sourceFile;
+
+  private BatchReportWriter writer;
+
+  @Before
+  public void prepare() throws IOException {
+    Project p = new Project("foo").setAnalysisDate(new Date(1234567L));
+    ResourceCache resourceCache = new ResourceCache();
+    org.sonar.api.resources.Resource sampleFile = org.sonar.api.resources.File.create("src/Foo.php").setEffectiveKey("foo:src/Foo.php");
+    resourceCache.add(p, null).setSnapshot(new Snapshot().setId(2));
+    File baseDir = temp.newFolder();
+    sourceFile = new File(baseDir, "src/Foo.php");
+    resourceCache.add(sampleFile, null).setInputPath(new DefaultInputFile("foo", "src/Foo.php").setLines(5).setModuleBaseDir(baseDir.toPath()).setCharset(Charsets.ISO_8859_1));
+    publisher = new SourcePublisher(resourceCache);
+    File outputDir = temp.newFolder();
+    writer = new BatchReportWriter(outputDir);
+  }
+
+  @Test
+  public void publishEmptySource() throws Exception {
+    FileUtils.write(sourceFile, "", Charsets.ISO_8859_1);
+
+    publisher.publish(writer);
+
+    File out = writer.getSourceFile(2);
+    assertThat(FileUtils.readFileToString(out, Charsets.UTF_8)).isEqualTo("");
+  }
+
+  @Test
+  public void publishSourceWithLastEmptyLine() throws Exception {
+    FileUtils.write(sourceFile, "1\n2\n3\n4\n", Charsets.ISO_8859_1);
+
+    publisher.publish(writer);
+
+    File out = writer.getSourceFile(2);
+    assertThat(FileUtils.readFileToString(out, Charsets.UTF_8)).isEqualTo("1\n2\n3\n4\n");
+  }
+
+  @Test
+  public void publishSourceWithLastLineNotEmpty() throws Exception {
+    FileUtils.write(sourceFile, "1\n2\n3\n4\n5", Charsets.ISO_8859_1);
+
+    publisher.publish(writer);
+
+    File out = writer.getSourceFile(2);
+    assertThat(FileUtils.readFileToString(out, Charsets.UTF_8)).isEqualTo("1\n2\n3\n4\n5");
+  }
+
+  @Test
+  public void cleanLineEnds() throws Exception {
+    FileUtils.write(sourceFile, "\n2\r\n3\n4\r5", Charsets.ISO_8859_1);
+
+    publisher.publish(writer);
+
+    File out = writer.getSourceFile(2);
+    assertThat(FileUtils.readFileToString(out, Charsets.UTF_8)).isEqualTo("\n2\n3\n4\n5");
+  }
+}
diff --git a/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java b/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java
index 60f04a9f626..3114e054718 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java
@@ -117,9 +117,9 @@ public class DefaultSensorStorageTest {
     InputFile file = new DefaultInputFile("foo", "src/Foo.php");
 
     ArgumentCaptor<org.sonar.api.measures.Measure> argumentCaptor = ArgumentCaptor.forClass(org.sonar.api.measures.Measure.class);
-    File sonarFile = File.create("src/Foo.php");
+    Resource sonarFile = File.create("src/Foo.php").setEffectiveKey("foo:src/Foo.php");
+    resourceCache.add(sonarFile, null).setInputPath(file);
     when(sonarIndex.addMeasure(eq(sonarFile), argumentCaptor.capture())).thenReturn(null);
-    when(sonarIndex.getResource(sonarFile)).thenReturn(sonarFile);
     sensorStorage.store(new DefaultMeasure()
       .onFile(file)
       .forMetric(CoreMetrics.NCLOC)
@@ -137,12 +137,11 @@ public class DefaultSensorStorageTest {
     InputFile file = new DefaultInputFile("foo", "src/Foo.php");
 
     ArgumentCaptor<org.sonar.api.measures.Measure> argumentCaptor = ArgumentCaptor.forClass(org.sonar.api.measures.Measure.class);
-    File sonarFile = File.create("src/Foo.php");
+    Resource sonarFile = File.create("src/Foo.php").setEffectiveKey("foo:src/Foo.php");
+    resourceCache.add(sonarFile, null).setInputPath(file);
 
     when(sonarIndex.addMeasure(eq(sonarFile), argumentCaptor.capture())).thenReturn(null);
 
-    when(sonarIndex.getResource(sonarFile)).thenReturn(sonarFile);
-
     sensorStorage.store(new DefaultMeasure()
       .onFile(file)
       .forMetric(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION)
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
index 40a0cb87d89..ea7b38ce9c7 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
@@ -50,18 +50,18 @@ public class FileMetadata implements BatchComponent {
   private static final char LINE_FEED = '\n';
   private static final char CARRIAGE_RETURN = '\r';
 
-  private abstract static class CharHandler {
+  public abstract static class CharHandler {
 
-    void handleAll(char c) {
+    protected void handleAll(char c) {
     }
 
-    void handleIgnoreEoL(char c) {
+    protected void handleIgnoreEoL(char c) {
     }
 
-    void newLine() {
+    protected void newLine() {
     }
 
-    void eof() {
+    protected void eof() {
     }
   }
 
@@ -79,7 +79,7 @@ public class FileMetadata implements BatchComponent {
     }
 
     @Override
-    void handleAll(char c) {
+    protected void handleAll(char c) {
       if (this.lines == 0) {
         this.lines = 1;
       }
@@ -91,7 +91,7 @@ public class FileMetadata implements BatchComponent {
     }
 
     @Override
-    void newLine() {
+    protected void newLine() {
       lines++;
       if (!blankLine) {
         nonBlankLines++;
@@ -100,14 +100,14 @@ public class FileMetadata implements BatchComponent {
     }
 
     @Override
-    void handleIgnoreEoL(char c) {
+    protected void handleIgnoreEoL(char c) {
       if (!Character.isWhitespace(c)) {
         blankLine = false;
       }
     }
 
     @Override
-    void eof() {
+    protected void eof() {
       if (!blankLine) {
         nonBlankLines++;
       }
@@ -128,19 +128,19 @@ public class FileMetadata implements BatchComponent {
     private StringBuilder sb = new StringBuilder();
 
     @Override
-    void handleIgnoreEoL(char c) {
+    protected void handleIgnoreEoL(char c) {
       sb.append(c);
     }
 
     @Override
-    void newLine() {
+    protected void newLine() {
       sb.append(LINE_FEED);
       globalMd5Digest.update(sb.toString().getBytes(Charsets.UTF_8));
       sb.setLength(0);
     }
 
     @Override
-    void eof() {
+    protected void eof() {
       if (sb.length() > 0) {
         globalMd5Digest.update(sb.toString().getBytes(Charsets.UTF_8));
       }
@@ -163,21 +163,21 @@ public class FileMetadata implements BatchComponent {
     }
 
     @Override
-    void handleIgnoreEoL(char c) {
+    protected void handleIgnoreEoL(char c) {
       if (!Character.isWhitespace(c)) {
         sb.append(c);
       }
     }
 
     @Override
-    void newLine() {
+    protected void newLine() {
       consumer.consume(line, sb.length() > 0 ? lineMd5Digest.digest(sb.toString().getBytes(Charsets.UTF_8)) : null);
       sb.setLength(0);
       line++;
     }
 
     @Override
-    void eof() {
+    protected void eof() {
       consumer.consume(line, sb.length() > 0 ? lineMd5Digest.digest(sb.toString().getBytes(Charsets.UTF_8)) : null);
     }
 
@@ -193,17 +193,17 @@ public class FileMetadata implements BatchComponent {
     }
 
     @Override
-    void handleAll(char c) {
+    protected void handleAll(char c) {
       currentOriginalOffset++;
     }
 
     @Override
-    void newLine() {
+    protected void newLine() {
       originalLineOffsets.add(currentOriginalOffset);
     }
 
     @Override
-    void eof() {
+    protected void eof() {
       lastValidOffset = currentOriginalOffset;
     }
 
@@ -246,7 +246,7 @@ public class FileMetadata implements BatchComponent {
       lineOffsetCounter.getLastValidOffset());
   }
 
-  private static void readFile(File file, Charset encoding, CharHandler... handlers) {
+  public static void readFile(File file, Charset encoding, CharHandler... handlers) {
     try (BOMInputStream bomIn = new BOMInputStream(new FileInputStream(file),
       ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE);
       Reader reader = new BufferedReader(new InputStreamReader(bomIn, encoding))) {
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/measures/CoreMetrics.java b/sonar-plugin-api/src/main/java/org/sonar/api/measures/CoreMetrics.java
index 33dc4b35b71..bebf743c01c 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/measures/CoreMetrics.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/measures/CoreMetrics.java
@@ -24,6 +24,7 @@ import com.google.common.base.Predicate;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import org.sonar.api.resources.Scopes;
+import org.sonar.api.test.MutableTestPlan;
 import org.sonar.api.utils.SonarException;
 
 import javax.annotation.Nullable;
@@ -485,7 +486,15 @@ public final class CoreMetrics {
     .setOptimizedBestValue(true)
     .create();
 
+  /**
+   * @deprecated since 5.2 use {@link MutableTestPlan}
+   */
+  @Deprecated
   public static final String TEST_DATA_KEY = "test_data";
+  /**
+   * @deprecated since 5.2 use {@link MutableTestPlan}
+   */
+  @Deprecated
   public static final Metric<String> TEST_DATA = new Metric.Builder(TEST_DATA_KEY, "Unit tests details", Metric.ValueType.DATA)
     .setDescription("Unit tests details")
     .setDirection(Metric.DIRECTION_WORST)
@@ -580,12 +589,19 @@ public final class CoreMetrics {
     .setDeleteHistoricalData(true)
     .create();
 
+  /**
+   * 
+   * @deprecated since 5.2 soon to be removed
+   */
+  @Deprecated
   public static final String COVERAGE_LINE_HITS_DATA_KEY = "coverage_line_hits_data";
 
   /**
    * Key-value pairs, where key - is a number of line, and value - is a number of hits for this line.
    * Use {@link CoverageMeasuresBuilder} to build measure for this metric.
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final Metric<String> COVERAGE_LINE_HITS_DATA = new Metric.Builder(COVERAGE_LINE_HITS_DATA_KEY, "Coverage hits by line", Metric.ValueType.DATA)
     .setDomain(DOMAIN_TESTS)
     .setDeleteHistoricalData(true)
@@ -656,25 +672,37 @@ public final class CoreMetrics {
     .setDeleteHistoricalData(true)
     .create();
 
+  /**
+   * @deprecated since 5.2 soon to be removed
+   */
+  @Deprecated
   public static final String CONDITIONS_BY_LINE_KEY = "conditions_by_line";
 
   /**
    * Use {@link CoverageMeasuresBuilder} to build measure for this metric.
    *
    * @since 2.7
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final Metric<String> CONDITIONS_BY_LINE = new Metric.Builder(CONDITIONS_BY_LINE_KEY, "Conditions by line", Metric.ValueType.DATA)
     .setDomain(DOMAIN_TESTS)
     .setDeleteHistoricalData(true)
     .create();
 
+  /**
+   * @deprecated since 5.2 soon to be removed
+   */
+  @Deprecated
   public static final String COVERED_CONDITIONS_BY_LINE_KEY = "covered_conditions_by_line";
 
   /**
    * Use {@link CoverageMeasuresBuilder} to build measure for this metric.
    *
    * @since 2.7
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final Metric<String> COVERED_CONDITIONS_BY_LINE = new Metric.Builder(COVERED_CONDITIONS_BY_LINE_KEY, "Covered conditions by line", Metric.ValueType.DATA)
     .setDomain(DOMAIN_TESTS)
     .setDeleteHistoricalData(true)
@@ -823,12 +851,16 @@ public final class CoreMetrics {
 
   /**
    * @since 2.12
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final String IT_COVERAGE_LINE_HITS_DATA_KEY = "it_coverage_line_hits_data";
 
   /**
    * @since 2.12
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final Metric<String> IT_COVERAGE_LINE_HITS_DATA = new Metric.Builder(IT_COVERAGE_LINE_HITS_DATA_KEY, "IT coverage hits data", Metric.ValueType.DATA)
     .setDescription("Integration Tests Code coverage line hits data")
     .setDirection(Metric.DIRECTION_NONE)
@@ -940,12 +972,16 @@ public final class CoreMetrics {
 
   /**
    * @since 2.12
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final String IT_CONDITIONS_BY_LINE_KEY = "it_conditions_by_line";
 
   /**
    * @since 2.12
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final Metric<String> IT_CONDITIONS_BY_LINE = new Metric.Builder(IT_CONDITIONS_BY_LINE_KEY, "IT conditions by line", Metric.ValueType.DATA)
     .setDomain(DOMAIN_INTEGRATION_TESTS)
     .setDeleteHistoricalData(true)
@@ -953,12 +989,16 @@ public final class CoreMetrics {
 
   /**
    * @since 2.12
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final String IT_COVERED_CONDITIONS_BY_LINE_KEY = "it_covered_conditions_by_line";
 
   /**
    * @since 2.12
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final Metric<String> IT_COVERED_CONDITIONS_BY_LINE = new Metric.Builder(IT_COVERED_CONDITIONS_BY_LINE_KEY, "IT covered conditions by line", Metric.ValueType.DATA)
     .setDomain(DOMAIN_INTEGRATION_TESTS)
     .setDeleteHistoricalData(true)
@@ -1107,12 +1147,16 @@ public final class CoreMetrics {
 
   /**
    * @since 3.3
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final String OVERALL_COVERAGE_LINE_HITS_DATA_KEY = "overall_coverage_line_hits_data";
 
   /**
    * @since 3.3
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final Metric<String> OVERALL_COVERAGE_LINE_HITS_DATA = new Metric.Builder(OVERALL_COVERAGE_LINE_HITS_DATA_KEY, "Overall coverage hits by line",
     Metric.ValueType.DATA)
     .setDescription("Coverage hits by all tests and by line")
@@ -1227,12 +1271,16 @@ public final class CoreMetrics {
 
   /**
    * @since 3.3
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final String OVERALL_CONDITIONS_BY_LINE_KEY = "overall_conditions_by_line";
 
   /**
    * @since 3.3
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final Metric<String> OVERALL_CONDITIONS_BY_LINE = new Metric.Builder(OVERALL_CONDITIONS_BY_LINE_KEY, "Overall conditions by line", Metric.ValueType.DATA)
     .setDescription("Overall conditions by all tests and by line")
     .setDomain(DOMAIN_OVERALL_TESTS)
@@ -1241,12 +1289,16 @@ public final class CoreMetrics {
 
   /**
    * @since 3.3
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final String OVERALL_COVERED_CONDITIONS_BY_LINE_KEY = "overall_covered_conditions_by_line";
 
   /**
    * @since 3.3
+   * @deprecated since 5.2 soon to be removed
    */
+  @Deprecated
   public static final Metric<String> OVERALL_COVERED_CONDITIONS_BY_LINE = new Metric.Builder(OVERALL_COVERED_CONDITIONS_BY_LINE_KEY, "Overall covered branches by line",
     Metric.ValueType.DATA)
     .setDescription("Overall covered branches by all tests and by line")