SONAR-6989 Feed the file duplication index in the batch report

10 files changed, 148 insertions, 182 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java b/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java
index 31784aa1e80..34df18afaa8 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java
@@ -19,16 +19,15 @@
  */
 package org.sonar.batch.analysis;
 
-import org.sonar.batch.bootstrap.AbstractAnalysisMode;
-
-import org.sonar.batch.mediumtest.FakePluginInstaller;
-import org.sonar.batch.bootstrap.GlobalProperties;
+import java.util.Map;
+import javax.annotation.CheckForNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.AnalysisMode;
-
-import java.util.Map;
+import org.sonar.batch.bootstrap.AbstractAnalysisMode;
+import org.sonar.batch.bootstrap.GlobalProperties;
+import org.sonar.batch.mediumtest.FakePluginInstaller;
 
 /**
  * @since 4.0
@@ -99,6 +98,7 @@ public class DefaultAnalysisMode extends AbstractAnalysisMode implements Analysi
     }
   }
 
+  @CheckForNull
   private static String getPropertyWithFallback(Map<String, String> props1, Map<String, String> props2, String key) {
     if (props1.containsKey(key)) {
       return props1.get(key);
diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdComponents.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdComponents.java
index e2c5555d4e2..ffac2cba012 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdComponents.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdComponents.java
@@ -20,11 +20,8 @@
 package org.sonar.batch.cpd;
 
 import com.google.common.collect.ImmutableList;
-
 import java.util.List;
 
-import org.sonar.batch.cpd.index.IndexFactory;
-
 public final class CpdComponents {
 
   private CpdComponents() {
@@ -34,7 +31,6 @@ public final class CpdComponents {
     return ImmutableList.of(
       CpdSensor.class,
       CpdMappings.class,
-      IndexFactory.class,
       JavaCpdEngine.class,
       DefaultCpdEngine.class);
   }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java
index 93fc1726b46..a63b04f0006 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java
@@ -23,6 +23,13 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.CoreProperties;
@@ -33,24 +40,13 @@ import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.config.Settings;
-import org.sonar.api.resources.Project;
-import org.sonar.api.utils.SonarException;
-import org.sonar.batch.cpd.index.IndexFactory;
 import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.index.BatchComponentCache;
+import org.sonar.batch.report.ReportPublisher;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.index.CloneGroup;
 import org.sonar.duplications.internal.pmd.TokenizerBridge;
 
-import javax.annotation.Nullable;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
 public class DefaultCpdEngine extends CpdEngine {
 
   private static final Logger LOG = LoggerFactory.getLogger(DefaultCpdEngine.class);
@@ -60,22 +56,18 @@ public class DefaultCpdEngine extends CpdEngine {
    */
   private static final int TIMEOUT = 5 * 60;
 
-  private final IndexFactory indexFactory;
   private final CpdMappings mappings;
   private final FileSystem fs;
   private final Settings settings;
-  private final Project project;
+  private final ReportPublisher publisher;
+  private final BatchComponentCache batchComponentCache;
 
-  public DefaultCpdEngine(@Nullable Project project, IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings) {
-    this.project = project;
-    this.indexFactory = indexFactory;
+  public DefaultCpdEngine(CpdMappings mappings, FileSystem fs, Settings settings, ReportPublisher publisher, BatchComponentCache batchComponentCache) {
     this.mappings = mappings;
     this.fs = fs;
     this.settings = settings;
-  }
-
-  public DefaultCpdEngine(IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings) {
-    this(null, indexFactory, mappings, fs, settings);
+    this.publisher = publisher;
+    this.batchComponentCache = batchComponentCache;
   }
 
   @Override
@@ -97,14 +89,13 @@ public class DefaultCpdEngine extends CpdEngine {
     List<InputFile> sourceFiles = Lists.newArrayList(fs.inputFiles(p.and(
       p.hasType(InputFile.Type.MAIN),
       p.hasLanguage(languageKey),
-      p.doesNotMatchPathPatterns(cpdExclusions)
-      )));
+      p.doesNotMatchPathPatterns(cpdExclusions))));
     if (sourceFiles.isEmpty()) {
       return;
     }
 
     // Create index
-    SonarDuplicationsIndex index = indexFactory.create(project, languageKey);
+    SonarDuplicationsIndex index = new SonarDuplicationsIndex(publisher, batchComponentCache, settings);
     populateIndex(languageKey, sourceFiles, mapping, index);
 
     // Detect
@@ -129,7 +120,7 @@ public class DefaultCpdEngine extends CpdEngine {
           filtered = null;
           LOG.warn("Timeout during detection of duplications for " + inputFile, e);
         } catch (InterruptedException | ExecutionException e) {
-          throw new SonarException("Fail during detection of duplication for " + inputFile, e);
+          throw new IllegalStateException("Fail during detection of duplication for " + inputFile, e);
         }
 
         JavaCpdEngine.save(context, inputFile, filtered);
@@ -144,8 +135,8 @@ public class DefaultCpdEngine extends CpdEngine {
     for (InputFile inputFile : sourceFiles) {
       LOG.debug("Populating index from {}", inputFile);
       String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
-      List<Block> blocks2 = bridge.chunk(resourceEffectiveKey, inputFile.file());
-      index.insert(inputFile, blocks2);
+      List<Block> blocks = bridge.chunk(resourceEffectiveKey, inputFile.file());
+      index.insert(inputFile, blocks);
     }
   }
 
@@ -162,7 +153,7 @@ public class DefaultCpdEngine extends CpdEngine {
   static int getDefaultBlockSize(String languageKey) {
     if ("cobol".equals(languageKey)) {
       return 30;
-    } else if ("abap".equals(languageKey) || "natur".equals(languageKey)) {
+    } else if ("abap".equals(languageKey)) {
       return 20;
     } else {
       return 10;
@@ -173,9 +164,6 @@ public class DefaultCpdEngine extends CpdEngine {
   int getMinimumTokens(String languageKey) {
     int minimumTokens = settings.getInt("sonar.cpd." + languageKey + ".minimumTokens");
     if (minimumTokens == 0) {
-      minimumTokens = settings.getInt(CoreProperties.CPD_MINIMUM_TOKENS_PROPERTY);
-    }
-    if (minimumTokens == 0) {
       minimumTokens = 100;
     }
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java
index 97059241af5..1adca4f11ca 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java
@@ -22,6 +22,21 @@ package org.sonar.batch.cpd;
 
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
 import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,10 +51,9 @@ import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure;
 import org.sonar.api.config.Settings;
 import org.sonar.api.measures.CoreMetrics;
-import org.sonar.api.resources.Project;
-import org.sonar.api.utils.SonarException;
-import org.sonar.batch.cpd.index.IndexFactory;
 import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.index.BatchComponentCache;
+import org.sonar.batch.report.ReportPublisher;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.block.BlockChunker;
 import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm;
@@ -52,23 +66,6 @@ import org.sonar.duplications.statement.Statement;
 import org.sonar.duplications.statement.StatementChunker;
 import org.sonar.duplications.token.TokenChunker;
 
-import javax.annotation.Nullable;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
 public class JavaCpdEngine extends CpdEngine {
 
   private static final Logger LOG = LoggerFactory.getLogger(JavaCpdEngine.class);
@@ -83,20 +80,16 @@ public class JavaCpdEngine extends CpdEngine {
   private static final int MAX_CLONE_GROUP_PER_FILE = 100;
   private static final int MAX_CLONE_PART_PER_GROUP = 100;
 
-  private final IndexFactory indexFactory;
   private final FileSystem fs;
   private final Settings settings;
-  private final Project project;
+  private final ReportPublisher publisher;
+  private final BatchComponentCache batchComponentCache;
 
-  public JavaCpdEngine(@Nullable Project project, IndexFactory indexFactory, FileSystem fs, Settings settings) {
-    this.project = project;
-    this.indexFactory = indexFactory;
+  public JavaCpdEngine(FileSystem fs, Settings settings, ReportPublisher publisher, BatchComponentCache batchComponentCache) {
     this.fs = fs;
     this.settings = settings;
-  }
-
-  public JavaCpdEngine(IndexFactory indexFactory, FileSystem fs, Settings settings) {
-    this(null, indexFactory, fs, settings);
+    this.publisher = publisher;
+    this.batchComponentCache = batchComponentCache;
   }
 
   @Override
@@ -112,17 +105,16 @@ public class JavaCpdEngine extends CpdEngine {
     List<InputFile> sourceFiles = Lists.newArrayList(fs.inputFiles(p.and(
       p.hasType(InputFile.Type.MAIN),
       p.hasLanguage(languageKey),
-      p.doesNotMatchPathPatterns(cpdExclusions)
-      )));
+      p.doesNotMatchPathPatterns(cpdExclusions))));
     if (sourceFiles.isEmpty()) {
       return;
     }
-    SonarDuplicationsIndex index = createIndex(project, languageKey, sourceFiles);
+    SonarDuplicationsIndex index = createIndex(sourceFiles);
     detect(index, context, sourceFiles);
   }
 
-  private SonarDuplicationsIndex createIndex(@Nullable Project project, String language, Iterable<InputFile> sourceFiles) {
-    final SonarDuplicationsIndex index = indexFactory.create(project, language);
+  private SonarDuplicationsIndex createIndex(Iterable<InputFile> sourceFiles) {
+    final SonarDuplicationsIndex index = new SonarDuplicationsIndex(publisher, batchComponentCache, settings);
 
     TokenChunker tokenChunker = JavaTokenProducer.build();
     StatementChunker statementChunker = JavaStatementBuilder.build();
@@ -139,7 +131,7 @@ public class JavaCpdEngine extends CpdEngine {
         reader = new InputStreamReader(new FileInputStream(inputFile.file()), fs.encoding());
         statements = statementChunker.chunk(tokenChunker.chunk(reader));
       } catch (FileNotFoundException e) {
-        throw new SonarException("Cannot find file " + inputFile.file(), e);
+        throw new IllegalStateException("Cannot find file " + inputFile.file(), e);
       } finally {
         IOUtils.closeQuietly(reader);
       }
@@ -166,10 +158,8 @@ public class JavaCpdEngine extends CpdEngine {
         } catch (TimeoutException e) {
           clones = null;
           LOG.warn("Timeout during detection of duplications for " + inputFile, e);
-        } catch (InterruptedException e) {
-          throw new SonarException("Fail during detection of duplication for " + inputFile, e);
-        } catch (ExecutionException e) {
-          throw new SonarException("Fail during detection of duplication for " + inputFile, e);
+        } catch (InterruptedException | ExecutionException e) {
+          throw new IllegalStateException("Fail during detection of duplication for " + inputFile, e);
         }
 
         save(context, inputFile, clones);
@@ -209,20 +199,20 @@ public class JavaCpdEngine extends CpdEngine {
       .forMetric(CoreMetrics.DUPLICATED_FILES)
       .on(inputFile)
       .withValue(1))
-      .setFromCore()
-      .save();
+        .setFromCore()
+        .save();
     ((DefaultMeasure<Integer>) context.<Integer>newMeasure()
       .forMetric(CoreMetrics.DUPLICATED_LINES)
       .on(inputFile)
       .withValue(duplicatedLines))
-      .setFromCore()
-      .save();
+        .setFromCore()
+        .save();
     ((DefaultMeasure<Integer>) context.<Integer>newMeasure()
       .forMetric(CoreMetrics.DUPLICATED_BLOCKS)
       .on(inputFile)
       .withValue(duplicatedBlocks))
-      .setFromCore()
-      .save();
+        .setFromCore()
+        .save();
   }
 
   private static void saveDuplications(org.sonar.api.batch.sensor.SensorContext context, InputFile inputFile, Iterable<CloneGroup> duplications) {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/index/IndexFactory.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/index/IndexFactory.java
deleted file mode 100644
index 31a4b03ce55..00000000000
--- a/sonar-batch/src/main/java/org/sonar/batch/cpd/index/IndexFactory.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.cpd.index;
-
-import org.sonar.batch.analysis.DefaultAnalysisMode;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import javax.annotation.Nullable;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.BatchSide;
-import org.sonar.api.config.Settings;
-import org.sonar.api.resources.Project;
-
-@BatchSide
-public class IndexFactory {
-
-  private final Settings settings;
-  private final DefaultAnalysisMode mode;
-
-  public IndexFactory(DefaultAnalysisMode mode, Settings settings) {
-    this.mode = mode;
-    this.settings = settings;
-  }
-
-  public SonarDuplicationsIndex create(@Nullable Project project, String languageKey) {
-    return new SonarDuplicationsIndex();
-  }
-
-  @VisibleForTesting
-  boolean verifyCrossProject(@Nullable Project project, Logger logger) {
-    boolean crossProject = false;
-
-    if (settings.getBoolean(CoreProperties.CPD_CROSS_PROJECT)) {
-      if (mode.isIssues()) {
-        logger.info("Cross-project analysis disabled. Not supported in issues mode.");
-      } else if (StringUtils.isNotBlank(settings.getString(CoreProperties.PROJECT_BRANCH_PROPERTY))) {
-        logger.info("Cross-project analysis disabled. Not supported on project branches.");
-      } else if (project == null) {
-        // New sensor mode
-        logger.info("Cross-project analysis disabled. Not supported in new sensor mode.");
-      } else {
-        logger.info("Cross-project analysis enabled");
-        crossProject = true;
-      }
-    } else {
-      logger.info("Cross-project analysis disabled");
-    }
-    return crossProject;
-  }
-}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarDuplicationsIndex.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarDuplicationsIndex.java
index ab583d672dd..51a094a869e 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarDuplicationsIndex.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarDuplicationsIndex.java
@@ -19,8 +19,17 @@
  */
 package org.sonar.batch.cpd.index;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
 import java.util.Collection;
+import org.apache.commons.lang.StringUtils;
+import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.config.Settings;
+import org.sonar.batch.index.BatchComponentCache;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.batch.protocol.output.BatchReport.DuplicationBlock;
+import org.sonar.batch.report.ReportPublisher;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.block.ByteArray;
 import org.sonar.duplications.index.AbstractCloneIndex;
@@ -30,13 +39,46 @@ import org.sonar.duplications.index.PackedMemoryCloneIndex;
 public class SonarDuplicationsIndex extends AbstractCloneIndex {
 
   private final CloneIndex mem = new PackedMemoryCloneIndex();
+  private final ReportPublisher publisher;
+  private final BatchComponentCache batchComponentCache;
+  private final Settings settings;
+
+  public SonarDuplicationsIndex(ReportPublisher publisher, BatchComponentCache batchComponentCache, Settings settings) {
+    this.publisher = publisher;
+    this.batchComponentCache = batchComponentCache;
+    this.settings = settings;
+  }
 
   public void insert(InputFile inputFile, Collection<Block> blocks) {
+    if (isCrossProjectDuplicationEnabled(settings)) {
+      int id = batchComponentCache.get(inputFile).batchId();
+      final BatchReport.DuplicationBlock.Builder builder = BatchReport.DuplicationBlock.newBuilder();
+      publisher.getWriter().writeDuplicationBlocks(id, Iterables.transform(blocks, new Function<Block, BatchReport.DuplicationBlock>() {
+        @Override
+        public DuplicationBlock apply(Block input) {
+          builder.clear();
+          builder.setStartLine(input.getStartLine());
+          builder.setEndLine(input.getEndLine());
+          builder.setStartTokenIndex(input.getStartUnit());
+          builder.setEndTokenIndex(input.getEndUnit());
+          for (int i : input.getBlockHash().toIntArray()) {
+            builder.addHash(i);
+          }
+          return builder.build();
+        }
+      }));
+    }
     for (Block block : blocks) {
       mem.insert(block);
     }
   }
 
+  public static boolean isCrossProjectDuplicationEnabled(Settings settings) {
+    return settings.getBoolean(CoreProperties.CPD_CROSS_PROJECT)
+      // No cross project duplication for branches
+      && StringUtils.isBlank(settings.getString(CoreProperties.PROJECT_BRANCH_PROPERTY));
+  }
+
   public Collection<Block> getByInputFile(InputFile inputFile, String resourceKey) {
     return mem.getByResourceId(resourceKey);
   }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/TaskResult.java b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/TaskResult.java
index 04b5724b2c3..49c7f95736b 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/TaskResult.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/TaskResult.java
@@ -41,10 +41,8 @@ import org.sonar.api.batch.fs.TextPointer;
 import org.sonar.api.batch.fs.TextRange;
 import org.sonar.api.batch.fs.internal.DefaultInputDir;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.duplication.Duplication;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 import org.sonar.api.issue.Issue;
-import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.batch.issue.IssueCache;
 import org.sonar.batch.protocol.output.BatchReport;
 import org.sonar.batch.protocol.output.BatchReport.Component;
@@ -63,7 +61,6 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver {
   private static final Logger LOG = LoggerFactory.getLogger(TaskResult.class);
 
   private List<Issue> issues = new ArrayList<>();
-  private Map<String, List<Duplication>> duplications = new HashMap<>();
   private Map<String, InputFile> inputFiles = new HashMap<>();
   private Map<String, Component> reportComponents = new HashMap<>();
   private Map<String, InputDir> inputDirs = new HashMap<>();
@@ -86,7 +83,6 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver {
 
     storeFs(container);
 
-    storeDuplication(container);
   }
 
   private void storeReportComponents(int componentRef, String parentModuleKey, @Nullable String branch) {
@@ -106,13 +102,6 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver {
     return reader;
   }
 
-  private void storeDuplication(ProjectScanContainer container) {
-    DuplicationCache duplicationCache = container.getComponentByType(DuplicationCache.class);
-    for (String effectiveKey : duplicationCache.componentKeys()) {
-      duplications.put(effectiveKey, Lists.<Duplication>newArrayList(duplicationCache.byComponent(effectiveKey)));
-    }
-  }
-
   private void storeFs(ProjectScanContainer container) {
     InputPathCache inputFileCache = container.getComponentByType(InputPathCache.class);
     for (InputFile inputPath : inputFileCache.allFiles()) {
@@ -127,6 +116,10 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver {
     return issues;
   }
 
+  public Component getReportComponent(String key) {
+    return reportComponents.get(key);
+  }
+
   public List<BatchReport.Issue> issuesFor(InputPath inputPath) {
     List<BatchReport.Issue> result = Lists.newArrayList();
     int ref = reportComponents.get(key(inputPath)).getRef();
@@ -162,10 +155,6 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver {
     return inputDirs.get(relativePath);
   }
 
-  public List<Duplication> duplicationsFor(InputFile inputFile) {
-    return duplications.get(((DefaultInputFile) inputFile).key());
-  }
-
   public Map<String, List<BatchReport.Measure>> allMeasures() {
     Map<String, List<BatchReport.Measure>> result = new HashMap<>();
     for (Map.Entry<String, Component> component : reportComponents.entrySet()) {
@@ -226,6 +215,32 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver {
     return Collections.emptyList();
   }
 
+  public List<BatchReport.Duplication> duplicationsFor(InputFile file) {
+    List<BatchReport.Duplication> result = new ArrayList<>();
+    int ref = reportComponents.get(((DefaultInputFile) file).key()).getRef();
+    try (CloseableIterator<BatchReport.Duplication> it = getReportReader().readComponentDuplications(ref)) {
+      while (it.hasNext()) {
+        result.add(it.next());
+      }
+    } catch (Exception e) {
+      throw new IllegalStateException(e);
+    }
+    return result;
+  }
+
+  public List<BatchReport.DuplicationBlock> duplicationBlocksFor(InputFile file) {
+    List<BatchReport.DuplicationBlock> result = new ArrayList<>();
+    int ref = reportComponents.get(((DefaultInputFile) file).key()).getRef();
+    try (CloseableIterator<BatchReport.DuplicationBlock> it = getReportReader().readComponentDuplicationBlocks(ref)) {
+      while (it.hasNext()) {
+        result.add(it.next());
+      }
+    } catch (Exception e) {
+      throw new IllegalStateException(e);
+    }
+    return result;
+  }
+
   @CheckForNull
   public BatchReport.Coverage coverageFor(InputFile file, int line) {
     int ref = reportComponents.get(((DefaultInputFile) file).key()).getRef();
diff --git a/sonar-batch/src/main/java/org/sonar/batch/report/DuplicationsPublisher.java b/sonar-batch/src/main/java/org/sonar/batch/report/DuplicationsPublisher.java
index 6cf132a9f6e..068cb1c10e4 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/report/DuplicationsPublisher.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/report/DuplicationsPublisher.java
@@ -81,7 +81,8 @@ public class DuplicationsPublisher implements ReportPublisherStep {
         if (sameProjectComponent != null) {
           blockBuilder.setOtherFileRef(sameProjectComponent.batchId());
         } else {
-          blockBuilder.setOtherFileKey(componentKey);
+          // Should never happens
+          throw new IllegalStateException("No cross project duplication supported on batch side: " + componentKey);
         }
       }
       dupBuilder.addDuplicate(blockBuilder
diff --git a/sonar-batch/src/main/java/org/sonar/batch/report/MetadataPublisher.java b/sonar-batch/src/main/java/org/sonar/batch/report/MetadataPublisher.java
index 52e98b53d81..9c1d2faca6d 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/report/MetadataPublisher.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/report/MetadataPublisher.java
@@ -21,7 +21,9 @@ package org.sonar.batch.report;
 
 import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
+import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Project;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
 import org.sonar.batch.index.BatchComponent;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.batch.protocol.output.BatchReport;
@@ -32,10 +34,12 @@ public class MetadataPublisher implements ReportPublisherStep {
 
   private final BatchComponentCache componentCache;
   private final ImmutableProjectReactor reactor;
+  private final Settings settings;
 
-  public MetadataPublisher(BatchComponentCache componentCache, ImmutableProjectReactor reactor) {
+  public MetadataPublisher(BatchComponentCache componentCache, ImmutableProjectReactor reactor, Settings settings) {
     this.componentCache = componentCache;
     this.reactor = reactor;
+    this.settings = settings;
   }
 
   @Override
@@ -46,6 +50,7 @@ public class MetadataPublisher implements ReportPublisherStep {
       .setAnalysisDate(((Project) rootProject.resource()).getAnalysisDate().getTime())
       // Here we want key without branch
       .setProjectKey(root.getKey())
+      .setCrossProjectDuplicationActivated(SonarDuplicationsIndex.isCrossProjectDuplicationEnabled(settings))
       .setRootComponentRef(rootProject.batchId());
     String branch = root.properties().get(CoreProperties.PROJECT_BRANCH_PROPERTY);
     if (branch != null) {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java b/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java
index 0c6a9d67fa2..bcb38b4958a 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java
@@ -95,18 +95,18 @@ public class DefaultSensorStorage implements SensorStorage {
   private final ModuleIssues moduleIssues;
   private final CoverageExclusions coverageExclusions;
   private final DuplicationCache duplicationCache;
-  private final BatchComponentCache resourceCache;
+  private final BatchComponentCache componentCache;
   private final ReportPublisher reportPublisher;
   private final MeasureCache measureCache;
 
   public DefaultSensorStorage(MetricFinder metricFinder, ModuleIssues moduleIssues,
     Settings settings, FileSystem fs, ActiveRules activeRules, DuplicationCache duplicationCache,
-    CoverageExclusions coverageExclusions, BatchComponentCache resourceCache, ReportPublisher reportPublisher, MeasureCache measureCache) {
+    CoverageExclusions coverageExclusions, BatchComponentCache componentCache, ReportPublisher reportPublisher, MeasureCache measureCache) {
     this.metricFinder = metricFinder;
     this.moduleIssues = moduleIssues;
     this.coverageExclusions = coverageExclusions;
     this.duplicationCache = duplicationCache;
-    this.resourceCache = resourceCache;
+    this.componentCache = componentCache;
     this.reportPublisher = reportPublisher;
     this.measureCache = measureCache;
   }
@@ -127,7 +127,7 @@ public class DefaultSensorStorage implements SensorStorage {
     setValueAccordingToMetricType(newMeasure, m, measureToSave);
     measureToSave.setFromCore(measure.isFromCore());
     InputComponent inputComponent = newMeasure.inputComponent();
-    Resource resource = resourceCache.get(inputComponent).resource();
+    Resource resource = componentCache.get(inputComponent).resource();
     if (coverageExclusions.accept(resource, measureToSave)) {
       saveMeasure(resource, measureToSave);
     }
@@ -183,7 +183,7 @@ public class DefaultSensorStorage implements SensorStorage {
   }
 
   private File getFile(InputFile file) {
-    BatchComponent r = resourceCache.get(file);
+    BatchComponent r = componentCache.get(file);
     if (r == null) {
       throw new IllegalStateException("Provided input file is not indexed");
     }
@@ -199,13 +199,13 @@ public class DefaultSensorStorage implements SensorStorage {
   public void store(DefaultHighlighting highlighting) {
     BatchReportWriter writer = reportPublisher.getWriter();
     DefaultInputFile inputFile = (DefaultInputFile) highlighting.inputFile();
-    writer.writeComponentSyntaxHighlighting(resourceCache.get(inputFile).batchId(),
+    writer.writeComponentSyntaxHighlighting(componentCache.get(inputFile).batchId(),
       Iterables.transform(highlighting.getSyntaxHighlightingRuleSet(), new BuildSyntaxHighlighting()));
   }
 
   public void store(DefaultInputFile inputFile, Map<Symbol, Set<TextRange>> referencesBySymbol) {
     BatchReportWriter writer = reportPublisher.getWriter();
-    writer.writeComponentSymbols(resourceCache.get(inputFile).batchId(),
+    writer.writeComponentSymbols(componentCache.get(inputFile).batchId(),
       Iterables.transform(referencesBySymbol.entrySet(), new Function<Map.Entry<Symbol, Set<TextRange>>, BatchReport.Symbol>() {
         private BatchReport.Symbol.Builder builder = BatchReport.Symbol.newBuilder();
         private BatchReport.TextRange.Builder rangeBuilder = BatchReport.TextRange.newBuilder();