SONAR-6317 Feed SCM in compute report - batch side

10 files changed, 162 insertions, 300 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java
index ff223b4b9f4..33f6246ea8c 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java
@@ -70,11 +70,7 @@ public class DefaultIndex extends SonarIndex {
     CoreMetrics.FILE_FEEDBACK_EDGES,
     CoreMetrics.FILE_TANGLE_INDEX,
     CoreMetrics.FILE_TANGLES,
-    // Computed by ScmSensor
-    CoreMetrics.SCM_AUTHORS_BY_LINE,
-    CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE,
-    CoreMetrics.SCM_REVISIONS_BY_LINE,
-    // Computed by core duplication plugin
+    // Computed by CpdSensor
     CoreMetrics.DUPLICATIONS_DATA,
     CoreMetrics.DUPLICATION_LINES_DATA,
     CoreMetrics.DUPLICATED_FILES,
diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/ResourceCache.java b/sonar-batch/src/main/java/org/sonar/batch/index/ResourceCache.java
index 4a97a6c8ab8..3b1ec2fe187 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/ResourceCache.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/ResourceCache.java
@@ -23,6 +23,8 @@ import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.Maps;
 import org.sonar.api.BatchComponent;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.resources.Library;
 import org.sonar.api.resources.Resource;
 
@@ -54,6 +56,10 @@ public class ResourceCache implements BatchComponent {
     }
   }
 
+  public BatchResource get(InputFile inputFile) {
+    return resources.get(((DefaultInputFile) inputFile).key());
+  }
+
   public BatchResource add(Resource resource, @Nullable Resource parentResource) {
     String componentKey = resource.getEffectiveKey();
     Preconditions.checkState(!Strings.isNullOrEmpty(componentKey), "Missing resource effective key");
diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/SourceDataFactory.java b/sonar-batch/src/main/java/org/sonar/batch/index/SourceDataFactory.java
index 2aa0b854e01..d2197170309 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/SourceDataFactory.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/SourceDataFactory.java
@@ -30,10 +30,13 @@ import org.sonar.api.batch.sensor.highlighting.internal.SyntaxHighlightingRule;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Measure;
 import org.sonar.api.source.Symbol;
-import org.sonar.api.utils.DateUtils;
 import org.sonar.api.utils.KeyValueFormat;
 import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.batch.highlighting.SyntaxHighlightingData;
+import org.sonar.batch.protocol.output.BatchReport.Scm;
+import org.sonar.batch.protocol.output.BatchReport.Scm.Changeset;
+import org.sonar.batch.protocol.output.BatchReportReader;
+import org.sonar.batch.report.PublishReportJob;
 import org.sonar.batch.scan.filesystem.InputFileMetadata;
 import org.sonar.batch.scan.measure.MeasureCache;
 import org.sonar.batch.source.CodeColorizers;
@@ -41,6 +44,7 @@ import org.sonar.batch.symbol.SymbolData;
 import org.sonar.core.source.SnapshotDataTypes;
 import org.sonar.core.source.db.FileSourceDto;
 import org.sonar.server.source.db.FileSourceDb;
+import org.sonar.server.source.db.FileSourceDb.Data.Builder;
 
 import java.io.IOException;
 import java.util.*;
@@ -57,18 +61,24 @@ public class SourceDataFactory implements BatchComponent {
   private final ComponentDataCache componentDataCache;
   private final DuplicationCache duplicationCache;
   private final CodeColorizers codeColorizers;
+  private final PublishReportJob publishReportJob;
+
+  private final ResourceCache resourceCache;
 
   public SourceDataFactory(MeasureCache measureCache, ComponentDataCache componentDataCache,
-    DuplicationCache duplicationCache, CodeColorizers codeColorizers) {
+    DuplicationCache duplicationCache, CodeColorizers codeColorizers, PublishReportJob publishReportJob, ResourceCache resourceCache) {
     this.measureCache = measureCache;
     this.componentDataCache = componentDataCache;
     this.duplicationCache = duplicationCache;
     this.codeColorizers = codeColorizers;
+    this.publishReportJob = publishReportJob;
+    this.resourceCache = resourceCache;
   }
 
   public byte[] consolidateData(DefaultInputFile inputFile, InputFileMetadata metadata) throws IOException {
     FileSourceDb.Data.Builder dataBuilder = createForSource(inputFile);
     applyLineMeasures(inputFile, dataBuilder);
+    applyScm(inputFile, dataBuilder);
     applyDuplications(inputFile.key(), dataBuilder);
     applyHighlighting(inputFile, metadata, dataBuilder);
     applySymbolReferences(inputFile, metadata, dataBuilder);
@@ -90,25 +100,30 @@ public class SourceDataFactory implements BatchComponent {
     return result;
   }
 
-  void applyLineMeasures(DefaultInputFile file, FileSourceDb.Data.Builder dataBuilder) {
-    applyLineMeasure(file.key(), CoreMetrics.SCM_AUTHORS_BY_LINE_KEY, dataBuilder, new MeasureOperation() {
-      @Override
-      public void apply(String value, FileSourceDb.Line.Builder lineBuilder) {
-        lineBuilder.setScmAuthor(value);
-      }
-    });
-    applyLineMeasure(file.key(), CoreMetrics.SCM_REVISIONS_BY_LINE_KEY, dataBuilder, new MeasureOperation() {
-      @Override
-      public void apply(String value, FileSourceDb.Line.Builder lineBuilder) {
-        lineBuilder.setScmRevision(value);
-      }
-    });
-    applyLineMeasure(file.key(), CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE_KEY, dataBuilder, new MeasureOperation() {
-      @Override
-      public void apply(String value, FileSourceDb.Line.Builder lineBuilder) {
-        lineBuilder.setScmDate(DateUtils.parseDateTimeQuietly(value).getTime());
+  void applyScm(DefaultInputFile inputFile, Builder dataBuilder) {
+    BatchReportReader reader = new BatchReportReader(publishReportJob.getReportDir());
+    Scm componentScm = reader.readComponentScm(resourceCache.get(inputFile).batchId());
+    if (componentScm != null) {
+      for (int i = 0; i < componentScm.getChangesetIndexByLineCount(); i++) {
+        int index = componentScm.getChangesetIndexByLine(i);
+        Changeset changeset = componentScm.getChangeset(index);
+        if (i < dataBuilder.getLinesCount()) {
+          FileSourceDb.Line.Builder lineBuilder = dataBuilder.getLinesBuilder(i);
+          if (changeset.hasAuthor()) {
+            lineBuilder.setScmAuthor(changeset.getAuthor());
+          }
+          if (changeset.hasRevision()) {
+            lineBuilder.setScmRevision(changeset.getRevision());
+          }
+          if (changeset.hasDate()) {
+            lineBuilder.setScmDate(changeset.getDate());
+          }
+        }
       }
-    });
+    }
+  }
+
+  void applyLineMeasures(DefaultInputFile file, FileSourceDb.Data.Builder dataBuilder) {
     applyLineMeasure(file.key(), CoreMetrics.COVERAGE_LINE_HITS_DATA_KEY, dataBuilder, new MeasureOperation() {
       @Override
       public void apply(String value, FileSourceDb.Line.Builder lineBuilder) {
@@ -167,17 +182,16 @@ public class SourceDataFactory implements BatchComponent {
 
   void applyLineMeasure(String inputFileKey, String metricKey, FileSourceDb.Data.Builder to, MeasureOperation op) {
     Iterable<Measure> measures = measureCache.byMetric(inputFileKey, metricKey);
-    if (measures != null) {
-      for (Measure measure : measures) {
-        Map<Integer, String> lineMeasures = KeyValueFormat.parseIntString((String) measure.value());
-        for (Map.Entry<Integer, String> lineMeasure : lineMeasures.entrySet()) {
-          int lineIdx = lineMeasure.getKey();
-          if (lineIdx <= to.getLinesCount()) {
-            String value = lineMeasure.getValue();
-            if (StringUtils.isNotEmpty(value)) {
-              FileSourceDb.Line.Builder lineBuilder = to.getLinesBuilder(lineIdx - 1);
-              op.apply(value, lineBuilder);
-            }
+    if (measures.iterator().hasNext()) {
+      Measure measure = measures.iterator().next();
+      Map<Integer, String> lineMeasures = KeyValueFormat.parseIntString((String) measure.value());
+      for (Map.Entry<Integer, String> lineMeasure : lineMeasures.entrySet()) {
+        int lineIdx = lineMeasure.getKey();
+        if (lineIdx <= to.getLinesCount()) {
+          String value = lineMeasure.getValue();
+          if (StringUtils.isNotEmpty(value)) {
+            FileSourceDb.Line.Builder lineBuilder = to.getLinesBuilder(lineIdx - 1);
+            op.apply(value, lineBuilder);
           }
         }
       }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java
index ed0142bcace..6200c172993 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java
@@ -36,6 +36,7 @@ import org.sonar.batch.bootstrapper.EnvironmentInformation;
 import org.sonar.batch.issue.tracking.ServerLineHashesLoader;
 import org.sonar.batch.protocol.input.*;
 import org.sonar.batch.protocol.input.BatchInput.ServerIssue;
+import org.sonar.batch.report.PublishReportJob;
 import org.sonar.batch.repository.GlobalRepositoriesLoader;
 import org.sonar.batch.repository.ProjectRepositoriesLoader;
 import org.sonar.batch.repository.ServerIssuesLoader;
@@ -61,6 +62,7 @@ public class BatchMediumTester {
   public static BatchMediumTesterBuilder builder() {
     BatchMediumTesterBuilder builder = new BatchMediumTesterBuilder().registerCoreMetrics();
     builder.bootstrapProperties.put(MEDIUM_TEST_ENABLED, "true");
+    builder.bootstrapProperties.put(PublishReportJob.KEEP_REPORT_PROP_KEY, "true");
     builder.bootstrapProperties.put(CoreProperties.WORKING_DIRECTORY, Files.createTempDir().getAbsolutePath());
     return builder;
   }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/report/PublishReportJob.java b/sonar-batch/src/main/java/org/sonar/batch/report/PublishReportJob.java
index 3a70259bede..9fe5d4bbd8b 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/report/PublishReportJob.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/report/PublishReportJob.java
@@ -22,13 +22,16 @@ package org.sonar.batch.report;
 import com.github.kevinsawicki.http.HttpRequest;
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.io.FileUtils;
+import org.codehaus.plexus.personality.plexus.lifecycle.phase.Startable;
+import org.codehaus.plexus.personality.plexus.lifecycle.phase.StartingException;
+import org.codehaus.plexus.personality.plexus.lifecycle.phase.StoppingException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.BatchComponent;
 import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.bootstrap.ProjectReactor;
 import org.sonar.api.config.Settings;
 import org.sonar.api.platform.Server;
-import org.sonar.api.resources.Project;
 import org.sonar.api.utils.TempFolder;
 import org.sonar.api.utils.ZipUtils;
 import org.sonar.batch.bootstrap.DefaultAnalysisMode;
@@ -40,33 +43,55 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 
-public class PublishReportJob implements BatchComponent {
+public class PublishReportJob implements BatchComponent, Startable {
 
   private static final Logger LOG = LoggerFactory.getLogger(PublishReportJob.class);
+  public static final String KEEP_REPORT_PROP_KEY = "sonar.batch.keepReport";
 
   private final ServerClient serverClient;
   private final Server server;
   private final Settings settings;
-  private final Project project;
+  private final ProjectReactor projectReactor;
   private final DefaultAnalysisMode analysisMode;
   private final TempFolder temp;
 
   private ReportPublisher[] publishers;
 
+  private File reportDir;
+  private BatchReportWriter writer;
+
   public PublishReportJob(Settings settings, ServerClient serverClient, Server server,
-    Project project, DefaultAnalysisMode analysisMode, TempFolder temp, ReportPublisher[] publishers) {
+    ProjectReactor projectReactor, DefaultAnalysisMode analysisMode, TempFolder temp, ReportPublisher[] publishers) {
     this.serverClient = serverClient;
     this.server = server;
-    this.project = project;
+    this.projectReactor = projectReactor;
     this.settings = settings;
     this.analysisMode = analysisMode;
     this.temp = temp;
     this.publishers = publishers;
   }
 
-  public PublishReportJob(Settings settings, ServerClient serverClient, Server server,
-    Project project, DefaultAnalysisMode analysisMode, TempFolder temp) {
-    this(settings, serverClient, server, project, analysisMode, temp, new ReportPublisher[0]);
+  @Override
+  public void start() throws StartingException {
+    reportDir = new File(projectReactor.getRoot().getWorkDir(), "batch-report");
+    writer = new BatchReportWriter(reportDir);
+  }
+
+  @Override
+  public void stop() throws StoppingException {
+    if (!settings.getBoolean(KEEP_REPORT_PROP_KEY)) {
+      FileUtils.deleteQuietly(reportDir);
+    } else {
+      LOG.info("Batch report generated in " + reportDir);
+    }
+  }
+
+  public File getReportDir() {
+    return reportDir;
+  }
+
+  public BatchReportWriter getWriter() {
+    return writer;
   }
 
   public void execute() {
@@ -83,8 +108,6 @@ public class PublishReportJob implements BatchComponent {
   private File prepareReport() {
     try {
       long startTime = System.currentTimeMillis();
-      File reportDir = temp.newDir("batch-report");
-      BatchReportWriter writer = new BatchReportWriter(reportDir);
       for (ReportPublisher publisher : publishers) {
         publisher.publish(writer);
       }
@@ -94,7 +117,6 @@ public class PublishReportJob implements BatchComponent {
       startTime = System.currentTimeMillis();
       File reportZip = temp.newFile("batch-report", ".zip");
       ZipUtils.zipDir(reportDir, reportZip);
-      FileUtils.deleteDirectory(reportDir);
       stopTime = System.currentTimeMillis();
       LOG.info("Analysis reports compressed in " + (stopTime - startTime) + "ms, zip size=" + FileUtils.byteCountToDisplaySize(FileUtils.sizeOf(reportZip)));
       return reportZip;
@@ -109,7 +131,8 @@ public class PublishReportJob implements BatchComponent {
     long startTime = System.currentTimeMillis();
     URL url;
     try {
-      url = new URL(serverClient.getURL() + "/api/computation/submit_report?projectKey=" + project.getEffectiveKey());
+      String effectiveKey = projectReactor.getRoot().getKeyWithBranch();
+      url = new URL(serverClient.getURL() + "/api/computation/submit_report?projectKey=" + effectiveKey);
     } catch (MalformedURLException e) {
       throw new IllegalArgumentException("Invalid URL", e);
     }
@@ -148,7 +171,8 @@ public class PublishReportJob implements BatchComponent {
       if (!baseUrl.endsWith("/")) {
         baseUrl += "/";
       }
-      String url = baseUrl + "dashboard/index/" + project.getKey();
+      String effectiveKey = projectReactor.getRoot().getKeyWithBranch();
+      String url = baseUrl + "dashboard/index/" + effectiveKey;
       logger.info("ANALYSIS SUCCESSFUL, you can browse {}", url);
       logger.info("Note that you will be able to access the updated dashboard once the server has processed the submitted analysis report.");
     }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectScmRepositoryLoader.java b/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectScmRepositoryLoader.java
deleted file mode 100644
index a5ee07dd685..00000000000
--- a/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectScmRepositoryLoader.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.repository;
-
-import com.google.common.collect.Maps;
-import org.sonar.api.BatchComponent;
-import org.sonar.api.batch.RequiresDB;
-import org.sonar.api.batch.bootstrap.ProjectDefinition;
-import org.sonar.api.batch.bootstrap.ProjectReactor;
-import org.sonar.api.database.DatabaseSession;
-import org.sonar.api.database.model.MeasureModel;
-import org.sonar.api.database.model.ResourceModel;
-import org.sonar.api.database.model.Snapshot;
-import org.sonar.api.measures.CoreMetrics;
-import org.sonar.api.measures.Metric;
-import org.sonar.api.resources.Qualifiers;
-import org.sonar.batch.protocol.input.FileData;
-import org.sonar.batch.protocol.input.ProjectRepositories;
-
-import javax.annotation.CheckForNull;
-import javax.persistence.NoResultException;
-import javax.persistence.Query;
-
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.sonar.api.utils.DateUtils.longToDate;
-
-/**
- * Waiting for SCM measure copy to be handled in computation stack we need to get previous measures from DB
- */
-@RequiresDB
-public class ProjectScmRepositoryLoader implements BatchComponent {
-
-  private final DatabaseSession session;
-  private final ProjectReactor reactor;
-  private final ProjectRepositories ref;
-
-  public ProjectScmRepositoryLoader(DatabaseSession session, ProjectReactor reactor, ProjectRepositories ref) {
-    this.session = session;
-    this.reactor = reactor;
-    this.ref = ref;
-  }
-
-  public void complete() {
-    for (ProjectDefinition module : reactor.getProjects()) {
-
-      for (Entry<String, FileData> fileDataByPaths : ref.fileDataByPath(module.getKeyWithBranch()).entrySet()) {
-        String path = fileDataByPaths.getKey();
-        FileData fileData = fileDataByPaths.getValue();
-        String lastCommits = null;
-        String revisions = null;
-        String authors = null;
-        List<Object[]> measuresByKey = query(module.getKeyWithBranch() + ":" + path, CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE_KEY, CoreMetrics.SCM_REVISIONS_BY_LINE_KEY,
-          CoreMetrics.SCM_AUTHORS_BY_LINE_KEY);
-        for (Object[] measureByKey : measuresByKey) {
-          if (measureByKey[0].equals(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE_KEY)) {
-            lastCommits = ((MeasureModel) measureByKey[1]).getData(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE);
-          } else if (measureByKey[0].equals(CoreMetrics.SCM_REVISIONS_BY_LINE_KEY)) {
-            revisions = ((MeasureModel) measureByKey[1]).getData(CoreMetrics.SCM_REVISIONS_BY_LINE);
-          } else if (measureByKey[0].equals(CoreMetrics.SCM_AUTHORS_BY_LINE_KEY)) {
-            authors = ((MeasureModel) measureByKey[1]).getData(CoreMetrics.SCM_AUTHORS_BY_LINE);
-          }
-        }
-        ref.addFileData(module.getKeyWithBranch(), path, new FileData(fileData.hash(), authors == null, lastCommits, revisions, authors));
-      }
-    }
-    ref.setLastAnalysisDate(lastSnapshotCreationDate(reactor.getRoot().getKeyWithBranch()));
-  }
-
-  private List<Object[]> query(String resourceKey, String... metricKeys) {
-    StringBuilder sb = new StringBuilder();
-    Map<String, Object> params = Maps.newHashMap();
-
-    sb.append("SELECT met.key, m");
-    sb.append(" FROM ")
-      .append(MeasureModel.class.getSimpleName())
-      .append(" m, ")
-      .append(Metric.class.getSimpleName())
-      .append(" met, ")
-      .append(ResourceModel.class.getSimpleName())
-      .append(" r, ")
-      .append(Snapshot.class.getSimpleName())
-      .append(" s WHERE met.id=m.metricId AND m.snapshotId=s.id AND s.resourceId=r.id AND r.key=:kee AND s.status=:status AND s.qualifier<>:lib");
-    params.put("kee", resourceKey);
-    params.put("status", Snapshot.STATUS_PROCESSED);
-    params.put("lib", Qualifiers.LIBRARY);
-
-    sb.append(" AND m.characteristicId IS NULL");
-    sb.append(" AND m.personId IS NULL");
-    sb.append(" AND m.ruleId IS NULL AND m.rulePriority IS NULL");
-    if (metricKeys.length > 0) {
-      sb.append(" AND met.key IN (:metricKeys) ");
-      params.put("metricKeys", Arrays.asList(metricKeys));
-    }
-    sb.append(" AND s.last=true ");
-    sb.append(" ORDER BY s.createdAt ");
-
-    Query jpaQuery = session.createQuery(sb.toString());
-
-    for (Map.Entry<String, Object> entry : params.entrySet()) {
-      jpaQuery.setParameter(entry.getKey(), entry.getValue());
-    }
-    return jpaQuery.getResultList();
-  }
-
-  @CheckForNull
-  Date lastSnapshotCreationDate(String resourceKey) {
-    StringBuilder sb = new StringBuilder();
-    Map<String, Object> params = Maps.newHashMap();
-
-    sb.append("SELECT s.buildDate");
-    sb.append(" FROM ")
-      .append(ResourceModel.class.getSimpleName())
-      .append(" r, ")
-      .append(Snapshot.class.getSimpleName())
-      .append(" s WHERE s.resourceId=r.id AND r.key=:kee AND s.status=:status AND s.qualifier<>:lib");
-    params.put("kee", resourceKey);
-    params.put("status", Snapshot.STATUS_PROCESSED);
-    params.put("lib", Qualifiers.LIBRARY);
-
-    sb.append(" AND s.last=true ");
-
-    Query jpaQuery = session.createQuery(sb.toString());
-
-    for (Map.Entry<String, Object> entry : params.entrySet()) {
-      jpaQuery.setParameter(entry.getKey(), entry.getValue());
-    }
-    try {
-      return longToDate((Long) jpaQuery.getSingleResult());
-    } catch (NoResultException e) {
-      return null;
-    }
-  }
-}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectReactorReady.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectReactorReady.java
index 45b4700e795..85ea965e514 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectReactorReady.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectReactorReady.java
@@ -22,7 +22,6 @@ package org.sonar.batch.scan;
 import org.sonar.api.batch.bootstrap.ProjectBuilder;
 import org.sonar.api.batch.bootstrap.ProjectReactor;
 import org.sonar.api.batch.bootstrap.internal.ProjectBuilderContext;
-import org.sonar.batch.repository.ProjectScmRepositoryLoader;
 
 import javax.annotation.Nullable;
 
@@ -43,27 +42,16 @@ public class ProjectReactorReady {
   private final ProjectBuilder[] projectBuilders;
   private final ProjectExclusions exclusions;
   private final ProjectReactorValidator validator;
-  private final ProjectScmRepositoryLoader projectScmRepositoryLoader;
 
-  public ProjectReactorReady(ProjectExclusions exclusions, ProjectReactor reactor, @Nullable ProjectBuilder[] projectBuilders, ProjectReactorValidator validator,
-    @Nullable ProjectScmRepositoryLoader projectScmRepositoryLoader) {
+  public ProjectReactorReady(ProjectExclusions exclusions, ProjectReactor reactor, @Nullable ProjectBuilder[] projectBuilders, ProjectReactorValidator validator) {
     this.exclusions = exclusions;
     this.reactor = reactor;
     this.projectBuilders = projectBuilders;
     this.validator = validator;
-    this.projectScmRepositoryLoader = projectScmRepositoryLoader;
-  }
-
-  public ProjectReactorReady(ProjectExclusions exclusions, ProjectReactor reactor, @Nullable ProjectBuilder[] projectBuilders, ProjectReactorValidator validator) {
-    this(exclusions, reactor, projectBuilders, validator, null);
-  }
-
-  public ProjectReactorReady(ProjectExclusions exclusions, ProjectReactor reactor, ProjectReactorValidator validator, ProjectScmRepositoryLoader projectScmRepositoryLoader) {
-    this(exclusions, reactor, new ProjectBuilder[0], validator, projectScmRepositoryLoader);
   }
 
   public ProjectReactorReady(ProjectExclusions exclusions, ProjectReactor reactor, ProjectReactorValidator validator) {
-    this(exclusions, reactor, new ProjectBuilder[0], validator, null);
+    this(exclusions, reactor, new ProjectBuilder[0], validator);
   }
 
   public void start() {
@@ -80,10 +68,5 @@ public class ProjectReactorReady {
     // 3 Validate final reactor
     validator.validate(reactor);
 
-    // 4 Complete missing SCM information from project repositories
-    if (projectScmRepositoryLoader != null) {
-      projectScmRepositoryLoader.complete();
-    }
-
   }
 }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
index ffa93f69285..aaa63f0d328 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
@@ -51,9 +51,8 @@ import org.sonar.batch.mediumtest.ScanTaskObservers;
 import org.sonar.batch.phases.GraphPersister;
 import org.sonar.batch.profiling.PhasesSumUpTimeProfiler;
 import org.sonar.batch.qualitygate.QualityGateProvider;
-import org.sonar.batch.report.EventCache;
+import org.sonar.batch.report.*;
 import org.sonar.batch.repository.ProjectRepositoriesProvider;
-import org.sonar.batch.repository.ProjectScmRepositoryLoader;
 import org.sonar.batch.repository.language.DefaultLanguagesRepository;
 import org.sonar.batch.rule.ActiveRulesProvider;
 import org.sonar.batch.rule.RulesProvider;
@@ -187,6 +186,12 @@ public class ProjectScanContainer extends ComponentContainer {
 
       ProjectSettings.class,
 
+      // Report
+      PublishReportJob.class,
+      ComponentsPublisher.class,
+      IssuesPublisher.class,
+      MeasuresPublisher.class,
+
       ScanTaskObservers.class);
   }
 
@@ -196,8 +201,6 @@ public class ProjectScanContainer extends ComponentContainer {
       SourcePersister.class,
       ResourceKeyMigration.class,
 
-      ProjectScmRepositoryLoader.class,
-
       // Users
       DefaultUserFinder.class,
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java b/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java
index b95ba5d2178..4323e5cff92 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java
@@ -20,25 +20,25 @@
 package org.sonar.batch.scm;
 
 import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.scm.BlameCommand.BlameOutput;
 import org.sonar.api.batch.scm.BlameLine;
-import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Metric;
 import org.sonar.api.measures.PropertiesBuilder;
-import org.sonar.api.utils.DateUtils;
+import org.sonar.batch.index.BatchResource;
+import org.sonar.batch.index.ResourceCache;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.batch.protocol.output.BatchReport.Scm.Builder;
+import org.sonar.batch.protocol.output.BatchReportWriter;
 import org.sonar.batch.util.ProgressReport;
 
 import javax.annotation.Nullable;
 
 import java.text.Normalizer;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 
@@ -49,14 +49,16 @@ class DefaultBlameOutput implements BlameOutput {
   private static final Pattern NON_ASCII_CHARS = Pattern.compile("[^\\x00-\\x7F]");
   private static final Pattern ACCENT_CODES = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");
 
-  private final SensorContext context;
+  private final BatchReportWriter writer;
+  private final ResourceCache componentCache;
   private final Set<InputFile> allFilesToBlame = new HashSet<InputFile>();
   private ProgressReport progressReport;
   private int count;
   private int total;
 
-  DefaultBlameOutput(SensorContext context, List<InputFile> filesToBlame) {
-    this.context = context;
+  DefaultBlameOutput(BatchReportWriter writer, ResourceCache componentCache, List<InputFile> filesToBlame) {
+    this.writer = writer;
+    this.componentCache = componentCache;
     this.allFilesToBlame.addAll(filesToBlame);
     count = 0;
     total = filesToBlame.size();
@@ -75,24 +77,45 @@ class DefaultBlameOutput implements BlameOutput {
       return;
     }
 
-    PropertiesBuilder<Integer, String> authors = propertiesBuilder(CoreMetrics.SCM_AUTHORS_BY_LINE);
-    PropertiesBuilder<Integer, String> dates = propertiesBuilder(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE);
-    PropertiesBuilder<Integer, String> revisions = propertiesBuilder(CoreMetrics.SCM_REVISIONS_BY_LINE);
+    BatchResource batchComponent = componentCache.get(file);
+    Builder scmBuilder = BatchReport.Scm.newBuilder();
+    scmBuilder.setComponentRef(batchComponent.batchId());
+    Map<String, Integer> changesetsIdByRevision = new HashMap<>();
 
-    int lineNumber = 1;
     for (BlameLine line : lines) {
-      authors.add(lineNumber, normalizeString(line.author()));
-      Date date = line.date();
-      dates.add(lineNumber, date != null ? DateUtils.formatDateTime(date) : "");
-      revisions.add(lineNumber, line.revision());
-      lineNumber++;
+      if (StringUtils.isNotBlank(line.revision())) {
+        Integer changesetId = changesetsIdByRevision.get(line.revision());
+        if (changesetId == null) {
+          addChangeset(scmBuilder, line);
+          changesetId = scmBuilder.getChangesetCount() - 1;
+          changesetsIdByRevision.put(line.revision(), changesetId);
+        }
+        scmBuilder.addChangesetIndexByLine(changesetId);
+      } else {
+        addChangeset(scmBuilder, line);
+      }
     }
-    ScmSensor.saveMeasures(context, file, authors.buildData(), dates.buildData(), revisions.buildData());
+    writer.writeComponentScm(scmBuilder.build());
     allFilesToBlame.remove(file);
     count++;
     progressReport.message(count + "/" + total + " files analyzed, last one was " + file.absolutePath());
   }
 
+  private void addChangeset(Builder scmBuilder, BlameLine line) {
+    BatchReport.Scm.Changeset.Builder changesetBuilder = BatchReport.Scm.Changeset.newBuilder();
+    if (StringUtils.isNotBlank(line.revision())) {
+      changesetBuilder.setRevision(line.revision());
+    }
+    if (StringUtils.isNotBlank(line.author())) {
+      changesetBuilder.setAuthor(normalizeString(line.author()));
+    }
+    Date date = line.date();
+    if (date != null) {
+      changesetBuilder.setDate(date.getTime());
+    }
+    scmBuilder.addChangeset(changesetBuilder.build());
+  }
+
   private String normalizeString(@Nullable String inputString) {
     if (inputString == null) {
       return "";
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java
index 09b74bc6a6a..700098ab142 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java
@@ -30,10 +30,10 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.sensor.Sensor;
 import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.batch.sensor.SensorDescriptor;
-import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure;
-import org.sonar.api.measures.CoreMetrics;
+import org.sonar.batch.index.ResourceCache;
 import org.sonar.batch.protocol.input.FileData;
 import org.sonar.batch.protocol.input.ProjectRepositories;
+import org.sonar.batch.report.PublishReportJob;
 import org.sonar.batch.scan.filesystem.InputFileMetadata;
 import org.sonar.batch.scan.filesystem.InputPathCache;
 
@@ -49,14 +49,19 @@ public final class ScmSensor implements Sensor {
   private final FileSystem fs;
   private final ProjectRepositories projectReferentials;
   private final InputPathCache inputPathCache;
+  private final ResourceCache resourceCache;
+  private final PublishReportJob publishReportJob;
 
   public ScmSensor(ProjectDefinition projectDefinition, ScmConfiguration configuration,
-    ProjectRepositories projectReferentials, FileSystem fs, InputPathCache inputPathCache) {
+    ProjectRepositories projectReferentials, FileSystem fs, InputPathCache inputPathCache, ResourceCache resourceCache,
+    PublishReportJob publishReportJob) {
     this.projectDefinition = projectDefinition;
     this.configuration = configuration;
     this.projectReferentials = projectReferentials;
     this.fs = fs;
     this.inputPathCache = inputPathCache;
+    this.resourceCache = resourceCache;
+    this.publishReportJob = publishReportJob;
   }
 
   @Override
@@ -66,9 +71,9 @@ public final class ScmSensor implements Sensor {
   }
 
   @Override
-  public void execute(final SensorContext context) {
+  public void execute(SensorContext context) {
     if (configuration.isDisabled()) {
-      LOG.info("SCM Sensor is disabled");
+      LOG.info("SCM Publisher is disabled");
       return;
     }
     if (configuration.provider() == null) {
@@ -76,51 +81,32 @@ public final class ScmSensor implements Sensor {
       return;
     }
 
-    List<InputFile> filesToBlame = collectFilesToBlame(context);
+    List<InputFile> filesToBlame = collectFilesToBlame();
     if (!filesToBlame.isEmpty()) {
       String key = configuration.provider().key();
       LOG.info("SCM provider for this project is: " + key);
-      DefaultBlameOutput output = new DefaultBlameOutput(context, filesToBlame);
+      DefaultBlameOutput output = new DefaultBlameOutput(publishReportJob.getWriter(), resourceCache, filesToBlame);
       configuration.provider().blameCommand().blame(new DefaultBlameInput(fs, filesToBlame), output);
       output.finish();
     }
   }
 
-  private List<InputFile> collectFilesToBlame(final SensorContext context) {
+  private List<InputFile> collectFilesToBlame() {
     if (configuration.forceReloadAll()) {
       LOG.warn("Forced reloading of SCM data for all files.");
     }
     List<InputFile> filesToBlame = new LinkedList<InputFile>();
-    for (InputFile f : fs.inputFiles(fs.predicates().all())) {
-      if (!configuration.forceReloadAll()) {
-        copyPreviousMeasuresForUnmodifiedFiles(context, filesToBlame, f);
-      } else {
-        filesToBlame.add(f);
-      }
-    }
-    return filesToBlame;
-  }
-
-  private void copyPreviousMeasuresForUnmodifiedFiles(final SensorContext context, List<InputFile> filesToBlame, InputFile f) {
-    FileData fileData = projectReferentials.fileData(projectDefinition.getKeyWithBranch(), f.relativePath());
-
-    if (f.status() == Status.SAME && fileData != null) {
-      if (fileData.needBlame()) {
+    for (InputFile f : inputPathCache.allFiles()) {
+      if (configuration.forceReloadAll()) {
         addIfNotEmpty(filesToBlame, (DefaultInputFile) f);
       } else {
-        // Copy previous measures
-        String scmAuthorsByLine = fileData.scmAuthorsByLine();
-        String scmLastCommitDatetimesByLine = fileData.scmLastCommitDatetimesByLine();
-        String scmRevisionsByLine = fileData.scmRevisionsByLine();
-        if (scmAuthorsByLine != null
-          && scmLastCommitDatetimesByLine != null
-          && scmRevisionsByLine != null) {
-          saveMeasures(context, f, scmAuthorsByLine, scmLastCommitDatetimesByLine, scmRevisionsByLine);
+        FileData fileData = projectReferentials.fileData(projectDefinition.getKeyWithBranch(), f.relativePath());
+        if (f.status() != Status.SAME || fileData == null || fileData.needBlame()) {
+          addIfNotEmpty(filesToBlame, (DefaultInputFile) f);
         }
       }
-    } else {
-      addIfNotEmpty(filesToBlame, (DefaultInputFile) f);
     }
+    return filesToBlame;
   }
 
   private void addIfNotEmpty(List<InputFile> filesToBlame, DefaultInputFile f) {
@@ -130,24 +116,4 @@ public final class ScmSensor implements Sensor {
     }
   }
 
-  static void saveMeasures(SensorContext context, InputFile f, String scmAuthorsByLine, String scmLastCommitDatetimesByLine, String scmRevisionsByLine) {
-    ((DefaultMeasure<String>) context.<String>newMeasure()
-      .onFile(f)
-      .forMetric(CoreMetrics.SCM_AUTHORS_BY_LINE)
-      .withValue(scmAuthorsByLine))
-      .setFromCore()
-      .save();
-    ((DefaultMeasure<String>) context.<String>newMeasure()
-      .onFile(f)
-      .forMetric(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE)
-      .withValue(scmLastCommitDatetimesByLine))
-      .setFromCore()
-      .save();
-    ((DefaultMeasure<String>) context.<String>newMeasure()
-      .onFile(f)
-      .forMetric(CoreMetrics.SCM_REVISIONS_BY_LINE)
-      .withValue(scmRevisionsByLine))
-      .setFromCore()
-      .save();
-  }
 }