SONAR-2867 Standard copy-paste detection should happen within a project, not only within a module

7 files changed, 425 insertions, 347 deletions

diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/AbstractCpdEngineTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/AbstractCpdEngineTest.java
deleted file mode 100644
index 1f843d8e284..00000000000
--- a/sonar-batch/src/test/java/org/sonar/batch/cpd/AbstractCpdEngineTest.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.cpd;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.resources.Project;
-import org.sonar.api.utils.log.LogTester;
-import org.sonar.api.utils.log.LoggerLevel;
-import org.sonar.batch.index.BatchComponent;
-import org.sonar.batch.index.BatchComponentCache;
-import org.sonar.batch.protocol.output.BatchReport.Duplication;
-import org.sonar.batch.protocol.output.BatchReportReader;
-import org.sonar.batch.protocol.output.BatchReportWriter;
-import org.sonar.batch.report.ReportPublisher;
-import org.sonar.core.util.CloseableIterator;
-import org.sonar.duplications.index.CloneGroup;
-import org.sonar.duplications.index.ClonePart;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class AbstractCpdEngineTest {
-
-  @Rule
-  public LogTester logTester = new LogTester();
-
-  @Rule
-  public TemporaryFolder temp = new TemporaryFolder();
-
-  private AbstractCpdEngine engine;
-
-  private BatchReportReader reader;
-  private DefaultInputFile inputFile1;
-  private BatchComponent batchComponent1;
-  private BatchComponent batchComponent2;
-  private BatchComponent batchComponent3;
-
-  @Before
-  public void before() throws IOException {
-    File outputDir = temp.newFolder();
-    ReportPublisher reportPublisher = mock(ReportPublisher.class);
-    when(reportPublisher.getWriter()).thenReturn(new BatchReportWriter(outputDir));
-    reader = new BatchReportReader(outputDir);
-    BatchComponentCache componentCache = new BatchComponentCache();
-    Project p = new Project("foo");
-    componentCache.add(p, null).setInputComponent(new DefaultInputModule("foo"));
-    org.sonar.api.resources.Resource sampleFile = org.sonar.api.resources.File.create("src/Foo.php").setEffectiveKey("foo:src/Foo.php");
-    inputFile1 = new DefaultInputFile("foo", "src/Foo.php").setLines(5);
-    batchComponent1 = componentCache.add(sampleFile, null).setInputComponent(inputFile1);
-    org.sonar.api.resources.Resource sampleFile2 = org.sonar.api.resources.File.create("src/Foo2.php").setEffectiveKey("foo:src/Foo2.php");
-    batchComponent2 = componentCache.add(sampleFile2, null).setInputComponent(new DefaultInputFile("foo", "src/Foo2.php").setLines(5));
-    org.sonar.api.resources.Resource sampleFile3 = org.sonar.api.resources.File.create("src/Foo3.php").setEffectiveKey("foo:src/Foo3.php");
-    batchComponent3 = componentCache.add(sampleFile3, null).setInputComponent(new DefaultInputFile("foo", "src/Foo3.php").setLines(5));
-    engine = new AbstractCpdEngine(reportPublisher, componentCache) {
-
-      @Override
-      boolean isLanguageSupported(String language) {
-        return false;
-      }
-
-      @Override
-      void analyse(String language, SensorContext context) {
-      }
-    };
-  }
-
-  @Test
-  public void testNothingToSave() {
-    engine.saveDuplications(inputFile1, Collections.EMPTY_LIST);
-
-    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(0);
-  }
-
-  @Test
-  public void testOneSimpleDuplicationBetweenTwoFiles() {
-    List<CloneGroup> groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 2, 4), new ClonePart(batchComponent2.key(), 0, 15, 17)));
-    engine.saveDuplications(inputFile1, groups);
-
-    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1);
-    CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
-    Duplication duplication = dups.next();
-    dups.close();
-    assertThat(duplication.getOriginPosition().getStartLine()).isEqualTo(2);
-    assertThat(duplication.getOriginPosition().getEndLine()).isEqualTo(4);
-    assertThat(duplication.getDuplicateList()).hasSize(1);
-    assertThat(duplication.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent2.batchId());
-    assertThat(duplication.getDuplicate(0).getRange().getStartLine()).isEqualTo(15);
-    assertThat(duplication.getDuplicate(0).getRange().getEndLine()).isEqualTo(17);
-  }
-
-  @Test
-  public void testDuplicationOnSameFile() throws Exception {
-    List<CloneGroup> groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent1.key(), 0, 215, 414)));
-    engine.saveDuplications(inputFile1, groups);
-
-    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1);
-    CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
-    Duplication duplication = dups.next();
-    dups.close();
-    assertThat(duplication.getOriginPosition().getStartLine()).isEqualTo(5);
-    assertThat(duplication.getOriginPosition().getEndLine()).isEqualTo(204);
-    assertThat(duplication.getDuplicateList()).hasSize(1);
-    assertThat(duplication.getDuplicate(0).hasOtherFileRef()).isFalse();
-    assertThat(duplication.getDuplicate(0).getRange().getStartLine()).isEqualTo(215);
-    assertThat(duplication.getDuplicate(0).getRange().getEndLine()).isEqualTo(414);
-  }
-
-  @Test
-  public void testTooManyDuplicates() throws Exception {
-    // 1 origin part + 101 duplicates = 102
-    List<ClonePart> parts = new ArrayList<>(AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP + 2);
-    for (int i = 0; i < AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP + 2; i++) {
-      parts.add(new ClonePart(batchComponent1.key(), i, i, i + 1));
-    }
-    List<CloneGroup> groups = Arrays.asList(CloneGroup.builder().setLength(0).setOrigin(parts.get(0)).setParts(parts).build());
-    engine.saveDuplications(inputFile1, groups);
-
-    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1);
-    CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
-    Duplication duplication = dups.next();
-    dups.close();
-    assertThat(duplication.getDuplicateList()).hasSize(AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP);
-
-    assertThat(logTester.logs(LoggerLevel.WARN)).contains("Too many duplication references on file " + inputFile1.relativePath() + " for block at line 0. Keep only the first "
-      + AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP + " references.");
-  }
-
-  @Test
-  public void testTooManyDuplications() throws Exception {
-    // 1 origin part + 101 duplicates = 102
-    List<CloneGroup> dups = new ArrayList<>(AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE + 1);
-    for (int i = 0; i < AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE + 1; i++) {
-      ClonePart clonePart = new ClonePart(batchComponent1.key(), i, i, i + 1);
-      ClonePart dupPart = new ClonePart(batchComponent1.key(), i + 1, i + 1, i + 2);
-      dups.add(newCloneGroup(clonePart, dupPart));
-    }
-    engine.saveDuplications(inputFile1, dups);
-
-    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE);
-
-    assertThat(logTester.logs(LoggerLevel.WARN))
-      .contains("Too many duplication groups on file " + inputFile1.relativePath() + ". Keep only the first " + AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE + " groups.");
-  }
-
-  @Test
-  public void testOneDuplicatedGroupInvolvingMoreThanTwoFiles() throws Exception {
-    List<CloneGroup> groups = Arrays
-      .asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 25, 224)));
-    engine.saveDuplications(inputFile1, groups);
-
-    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1);
-    CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
-    Duplication duplication = dups.next();
-    dups.close();
-    assertThat(duplication.getOriginPosition().getStartLine()).isEqualTo(5);
-    assertThat(duplication.getOriginPosition().getEndLine()).isEqualTo(204);
-    assertThat(duplication.getDuplicateList()).hasSize(2);
-    assertThat(duplication.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent2.batchId());
-    assertThat(duplication.getDuplicate(0).getRange().getStartLine()).isEqualTo(15);
-    assertThat(duplication.getDuplicate(0).getRange().getEndLine()).isEqualTo(214);
-    assertThat(duplication.getDuplicate(1).getOtherFileRef()).isEqualTo(batchComponent3.batchId());
-    assertThat(duplication.getDuplicate(1).getRange().getStartLine()).isEqualTo(25);
-    assertThat(duplication.getDuplicate(1).getRange().getEndLine()).isEqualTo(224);
-  }
-
-  @Test
-  public void testTwoDuplicatedGroupsInvolvingThreeFiles() throws Exception {
-    List<CloneGroup> groups = Arrays.asList(
-      newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214)),
-      newCloneGroup(new ClonePart(batchComponent1.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 15, 214)));
-    engine.saveDuplications(inputFile1, groups);
-
-    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(2);
-    CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
-    Duplication duplication1 = dups.next();
-    Duplication duplication2 = dups.next();
-    dups.close();
-    assertThat(duplication1.getOriginPosition().getStartLine()).isEqualTo(5);
-    assertThat(duplication1.getOriginPosition().getEndLine()).isEqualTo(204);
-    assertThat(duplication1.getDuplicateList()).hasSize(1);
-    assertThat(duplication1.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent2.batchId());
-    assertThat(duplication1.getDuplicate(0).getRange().getStartLine()).isEqualTo(15);
-    assertThat(duplication1.getDuplicate(0).getRange().getEndLine()).isEqualTo(214);
-
-    assertThat(duplication2.getOriginPosition().getStartLine()).isEqualTo(15);
-    assertThat(duplication2.getOriginPosition().getEndLine()).isEqualTo(214);
-    assertThat(duplication2.getDuplicateList()).hasSize(1);
-    assertThat(duplication2.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent3.batchId());
-    assertThat(duplication2.getDuplicate(0).getRange().getStartLine()).isEqualTo(15);
-    assertThat(duplication2.getDuplicate(0).getRange().getEndLine()).isEqualTo(214);
-  }
-
-  private CloneGroup newCloneGroup(ClonePart... parts) {
-    return CloneGroup.builder().setLength(0).setOrigin(parts[0]).setParts(Arrays.asList(parts)).build();
-  }
-
-}
diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdExecutorTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdExecutorTest.java
new file mode 100644
index 00000000000..4490a21e828
--- /dev/null
+++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdExecutorTest.java
@@ -0,0 +1,240 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.cpd;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.api.config.Settings;
+import org.sonar.api.resources.Project;
+import org.sonar.api.utils.log.LogTester;
+import org.sonar.api.utils.log.LoggerLevel;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.index.BatchComponent;
+import org.sonar.batch.index.BatchComponentCache;
+import org.sonar.batch.protocol.output.BatchReportReader;
+import org.sonar.batch.protocol.output.BatchReportWriter;
+import org.sonar.batch.protocol.output.BatchReport.Duplicate;
+import org.sonar.batch.protocol.output.BatchReport.Duplication;
+import org.sonar.batch.report.ReportPublisher;
+import org.sonar.core.util.CloseableIterator;
+import org.sonar.duplications.index.CloneGroup;
+import org.sonar.duplications.index.ClonePart;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CpdExecutorTest {
+  private CpdExecutor executor;
+  private Settings settings;
+  private SonarDuplicationsIndex index;
+  private ReportPublisher publisher;
+  private BatchComponentCache componentCache;
+
+  @Rule
+  public LogTester logTester = new LogTester();
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  // private AbstractCpdEngine engine;
+
+  private BatchReportReader reader;
+  private BatchComponent batchComponent1;
+  private BatchComponent batchComponent2;
+  private BatchComponent batchComponent3;
+
+  @Before
+  public void setUp() throws IOException {
+    File outputDir = temp.newFolder();
+
+    settings = new Settings();
+    index = mock(SonarDuplicationsIndex.class);
+    publisher = mock(ReportPublisher.class);
+    when(publisher.getWriter()).thenReturn(new BatchReportWriter(outputDir));
+    componentCache = new BatchComponentCache();
+    executor = new CpdExecutor(settings, index, publisher, componentCache);
+    reader = new BatchReportReader(outputDir);
+
+    Project p = new Project("foo");
+    componentCache.add(p, null).setInputComponent(new DefaultInputModule("foo"));
+
+    batchComponent1 = createComponent("src/Foo.php", 5);
+    batchComponent2 = createComponent("src/Foo2.php", 5);
+    batchComponent3 = createComponent("src/Foo3.php", 5);
+  }
+
+  private BatchComponent createComponent(String relativePath, int lines) {
+    org.sonar.api.resources.Resource sampleFile = org.sonar.api.resources.File.create("relativePath").setEffectiveKey("foo:" + relativePath);
+    return componentCache.add(sampleFile, null).setInputComponent(new DefaultInputFile("foo", relativePath).setLines(lines));
+  }
+
+  @Test
+  public void defaultMinimumTokens() {
+    assertThat(executor.getMinimumTokens("java")).isEqualTo(100);
+  }
+
+  @Test
+  public void minimumTokensByLanguage() {
+    settings.setProperty("sonar.cpd.java.minimumTokens", "42");
+    settings.setProperty("sonar.cpd.php.minimumTokens", "33");
+    assertThat(executor.getMinimumTokens("java")).isEqualTo(42);
+
+    settings.setProperty("sonar.cpd.java.minimumTokens", "42");
+    settings.setProperty("sonar.cpd.php.minimumTokens", "33");
+    assertThat(executor.getMinimumTokens("php")).isEqualTo(33);
+  }
+
+  @Test
+  public void testNothingToSave() {
+    executor.saveDuplications(batchComponent1, Collections.<CloneGroup>emptyList());
+    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(0);
+  }
+
+  @Test
+  public void reportOneSimpleDuplicationBetweenTwoFiles() {
+    List<CloneGroup> groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 2, 4), new ClonePart(batchComponent2.key(), 0, 15, 17)));
+
+    executor.saveDuplications(batchComponent1, groups);
+
+    Duplication[] dups = readDuplications(1);
+    assertDuplication(dups[0], 2, 4, batchComponent2.batchId(), 15, 17);
+  }
+
+  @Test
+  public void reportDuplicationOnSameFile() throws Exception {
+    List<CloneGroup> groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent1.key(), 0, 215, 414)));
+    executor.saveDuplications(batchComponent1, groups);
+
+    Duplication[] dups = readDuplications(1);
+    assertDuplication(dups[0], 5, 204, null, 215, 414);
+  }
+
+  @Test
+  public void reportTooManyDuplicates() throws Exception {
+    // 1 origin part + 101 duplicates = 102
+    List<ClonePart> parts = new ArrayList<>(CpdExecutor.MAX_CLONE_PART_PER_GROUP + 2);
+    for (int i = 0; i < CpdExecutor.MAX_CLONE_PART_PER_GROUP + 2; i++) {
+      parts.add(new ClonePart(batchComponent1.key(), i, i, i + 1));
+    }
+    List<CloneGroup> groups = Arrays.asList(CloneGroup.builder().setLength(0).setOrigin(parts.get(0)).setParts(parts).build());
+    executor.saveDuplications(batchComponent1, groups);
+
+    Duplication[] dups = readDuplications(1);
+    assertThat(dups[0].getDuplicateList()).hasSize(CpdExecutor.MAX_CLONE_PART_PER_GROUP);
+
+    assertThat(logTester.logs(LoggerLevel.WARN))
+      .contains("Too many duplication references on file " + batchComponent1.inputComponent() + " for block at line 0. Keep only the first "
+        + CpdExecutor.MAX_CLONE_PART_PER_GROUP + " references.");
+  }
+
+  @Test
+  public void reportTooManyDuplications() throws Exception {
+    // 1 origin part + 101 duplicates = 102
+    List<CloneGroup> dups = new ArrayList<>(CpdExecutor.MAX_CLONE_GROUP_PER_FILE + 1);
+    for (int i = 0; i < CpdExecutor.MAX_CLONE_GROUP_PER_FILE + 1; i++) {
+      ClonePart clonePart = new ClonePart(batchComponent1.key(), i, i, i + 1);
+      ClonePart dupPart = new ClonePart(batchComponent1.key(), i + 1, i + 1, i + 2);
+      dups.add(newCloneGroup(clonePart, dupPart));
+    }
+    executor.saveDuplications(batchComponent1, dups);
+
+    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(CpdExecutor.MAX_CLONE_GROUP_PER_FILE);
+
+    assertThat(logTester.logs(LoggerLevel.WARN))
+      .contains("Too many duplication groups on file " + batchComponent1.inputComponent() + ". Keep only the first " + CpdExecutor.MAX_CLONE_GROUP_PER_FILE + " groups.");
+  }
+
+  @Test
+  public void reportOneDuplicatedGroupInvolvingMoreThanTwoFiles() throws Exception {
+    List<CloneGroup> groups = Arrays
+      .asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 25, 224)));
+    executor.saveDuplications(batchComponent1, groups);
+
+    Duplication[] dups = readDuplications(1);
+    assertDuplication(dups[0], 5, 204, 2);
+    assertDuplicate(dups[0].getDuplicate(0), batchComponent2.batchId(), 15, 214);
+    assertDuplicate(dups[0].getDuplicate(1), batchComponent3.batchId(), 25, 224);
+  }
+
+  @Test
+  public void reportTwoDuplicatedGroupsInvolvingThreeFiles() throws Exception {
+    List<CloneGroup> groups = Arrays.asList(
+      newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214)),
+      newCloneGroup(new ClonePart(batchComponent1.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 15, 214)));
+    executor.saveDuplications(batchComponent1, groups);
+
+    Duplication[] dups = readDuplications(2);
+    assertDuplication(dups[0], 5, 204, batchComponent2.batchId(), 15, 214);
+    assertDuplication(dups[1], 15, 214, batchComponent3.batchId(), 15, 214);
+  }
+  
+  private Duplication[] readDuplications(int expected) {
+    assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(expected);
+    Duplication[] duplications = new Duplication[expected];
+    CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
+    
+    for(int i = 0; i< expected; i++) {
+      duplications[i] = dups.next();
+    }
+    dups.close();
+    return duplications;
+  }
+  
+  private void assertDuplicate(Duplicate d, int otherFileRef, int rangeStartLine, int rangeEndLine) {
+    assertThat(d.getOtherFileRef()).isEqualTo(otherFileRef);
+    assertThat(d.getRange().getStartLine()).isEqualTo(rangeStartLine);
+    assertThat(d.getRange().getEndLine()).isEqualTo(rangeEndLine);
+  }
+  
+  private void assertDuplication(Duplication d, int originStartLine, int originEndLine, int numDuplicates) {
+    assertThat(d.getOriginPosition().getStartLine()).isEqualTo(originStartLine);
+    assertThat(d.getOriginPosition().getEndLine()).isEqualTo(originEndLine);
+    assertThat(d.getDuplicateList()).hasSize(numDuplicates);
+  }
+  
+  private void assertDuplication(Duplication d, int originStartLine, int originEndLine, Integer otherFileRef, int rangeStartLine, int rangeEndLine) {
+    assertThat(d.getOriginPosition().getStartLine()).isEqualTo(originStartLine);
+    assertThat(d.getOriginPosition().getEndLine()).isEqualTo(originEndLine);
+    assertThat(d.getDuplicateList()).hasSize(1);
+    if(otherFileRef != null) {
+    assertThat(d.getDuplicate(0).getOtherFileRef()).isEqualTo(otherFileRef);
+    } else {
+      assertThat(d.getDuplicate(0).hasOtherFileRef()).isFalse();
+    }
+    assertThat(d.getDuplicate(0).getRange().getStartLine()).isEqualTo(rangeStartLine);
+    assertThat(d.getDuplicate(0).getRange().getEndLine()).isEqualTo(rangeEndLine);
+  }
+  
+  private CloneGroup newCloneGroup(ClonePart... parts) {
+    return CloneGroup.builder().setLength(0).setOrigin(parts[0]).setParts(Arrays.asList(parts)).build();
+  }
+}
diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java
index 5c5adb3a4c2..948f008918b 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java
@@ -36,15 +36,15 @@ public class CpdSensorTest {
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
 
-  JavaCpdEngine sonarEngine;
-  DefaultCpdEngine sonarBridgeEngine;
+  JavaCpdIndexer sonarEngine;
+  DefaultCpdIndexer sonarBridgeEngine;
   CpdSensor sensor;
   Settings settings;
 
   @Before
   public void setUp() throws IOException {
-    sonarEngine = new JavaCpdEngine(null, null, null, null);
-    sonarBridgeEngine = new DefaultCpdEngine(new CpdMappings(), null, null, null, null);
+    sonarEngine = new JavaCpdIndexer(null, null, null);
+    sonarBridgeEngine = new DefaultCpdIndexer(new CpdMappings(), null, null, null);
     settings = new Settings(new PropertyDefinitions(CpdComponents.class));
 
     DefaultFileSystem fs = new DefaultFileSystem(temp.newFolder().toPath());
diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdIndexerTest.java
index 427c62a1e4c..51f082f5f69 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdIndexerTest.java
@@ -31,15 +31,15 @@ import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
-public class DefaultCpdEngineTest {
+public class DefaultCpdIndexerTest {
 
-  private DefaultCpdEngine engine;
+  private DefaultCpdIndexer engine;
   private Settings settings;
 
   @Before
   public void init() {
     settings = new Settings();
-    engine = new DefaultCpdEngine(null, null, settings, null, null);
+    engine = new DefaultCpdIndexer(null, null, settings, null);
   }
 
   @Test
@@ -59,9 +59,9 @@ public class DefaultCpdEngineTest {
 
   @Test
   public void shouldReturnDefaultBlockSize() {
-    assertThat(DefaultCpdEngine.getDefaultBlockSize("cobol")).isEqualTo(30);
-    assertThat(DefaultCpdEngine.getDefaultBlockSize("abap")).isEqualTo(20);
-    assertThat(DefaultCpdEngine.getDefaultBlockSize("other")).isEqualTo(10);
+    assertThat(DefaultCpdIndexer.getDefaultBlockSize("cobol")).isEqualTo(30);
+    assertThat(DefaultCpdIndexer.getDefaultBlockSize("abap")).isEqualTo(20);
+    assertThat(DefaultCpdIndexer.getDefaultBlockSize("other")).isEqualTo(10);
   }
 
   @Test
@@ -77,20 +77,4 @@ public class DefaultCpdEngineTest {
     assertThat(engine.getBlockSize("cobol")).isEqualTo(42);
   }
 
-  @Test
-  public void defaultMinimumTokens() {
-    assertThat(engine.getMinimumTokens("java")).isEqualTo(100);
-  }
-
-  @Test
-  public void minimumTokensByLanguage() {
-    settings.setProperty("sonar.cpd.java.minimumTokens", "42");
-    settings.setProperty("sonar.cpd.php.minimumTokens", "33");
-    assertThat(engine.getMinimumTokens("java")).isEqualTo(42);
-
-    settings.setProperty("sonar.cpd.java.minimumTokens", "42");
-    settings.setProperty("sonar.cpd.php.minimumTokens", "33");
-    assertThat(engine.getMinimumTokens("php")).isEqualTo(33);
-  }
-
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java
deleted file mode 100644
index 798f821d1ab..00000000000
--- a/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.cpd;
-
-import java.io.File;
-import org.apache.commons.io.FileUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.sonar.api.batch.fs.internal.DefaultFileSystem;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.config.Settings;
-import org.sonar.batch.index.BatchComponentCache;
-import org.sonar.batch.protocol.output.BatchReport;
-import org.sonar.batch.protocol.output.BatchReport.Duplication;
-import org.sonar.batch.protocol.output.BatchReportReader;
-import org.sonar.batch.protocol.output.BatchReportWriter;
-import org.sonar.batch.report.ReportPublisher;
-import org.sonar.core.util.CloseableIterator;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class JavaCpdEngineTest {
-
-  private static final String JAVA = "java";
-  @Rule
-  public TemporaryFolder temp = new TemporaryFolder();
-
-  @Test
-  public void testJavaCodeWithTwoCloneGroupAtSameLines() throws Exception {
-
-    File baseDir = temp.newFolder();
-    DefaultFileSystem fs = new DefaultFileSystem(baseDir);
-    DefaultInputFile file = new DefaultInputFile("foo", "src/ManyStatements.java").setLanguage(JAVA);
-    fs.add(file);
-    BatchComponentCache batchComponentCache = new BatchComponentCache();
-    batchComponentCache.add(org.sonar.api.resources.File.create("src/Foo.java").setEffectiveKey("foo:src/ManyStatements.java"), null).setInputComponent(file);
-    File ioFile = file.file();
-    FileUtils.copyURLToFile(this.getClass().getResource("ManyStatements.java"), ioFile);
-
-    File reportOut = temp.newFolder();
-    ReportPublisher reportPublisher = mock(ReportPublisher.class);
-    when(reportPublisher.getWriter()).thenReturn(new BatchReportWriter(reportOut));
-    JavaCpdEngine engine = new JavaCpdEngine(fs, new Settings(), reportPublisher, batchComponentCache);
-    engine.analyse(JAVA, mock(SensorContext.class));
-
-    BatchReportReader reader = new BatchReportReader(reportOut);
-    try (CloseableIterator<BatchReport.Duplication> it = reader.readComponentDuplications(1)) {
-      Duplication dupGroup1 = it.next();
-      Duplication dupGroup2 = it.next();
-
-      assertThat(dupGroup1.getOriginPosition().getStartLine()).isEqualTo(6);
-      assertThat(dupGroup1.getOriginPosition().getEndLine()).isEqualTo(6);
-      assertThat(dupGroup1.getDuplicateCount()).isEqualTo(1);
-      assertThat(dupGroup1.getDuplicate(0).getRange().getStartLine()).isEqualTo(8);
-      assertThat(dupGroup1.getDuplicate(0).getRange().getEndLine()).isEqualTo(8);
-
-      assertThat(dupGroup2.getOriginPosition().getStartLine()).isEqualTo(6);
-      assertThat(dupGroup2.getOriginPosition().getEndLine()).isEqualTo(6);
-      assertThat(dupGroup2.getDuplicateCount()).isEqualTo(2);
-      assertThat(dupGroup2.getDuplicate(0).getRange().getStartLine()).isEqualTo(7);
-      assertThat(dupGroup2.getDuplicate(0).getRange().getEndLine()).isEqualTo(7);
-      assertThat(dupGroup2.getDuplicate(1).getRange().getStartLine()).isEqualTo(8);
-      assertThat(dupGroup2.getDuplicate(1).getRange().getEndLine()).isEqualTo(8);
-    } catch (Exception e) {
-      throw new IllegalStateException(e);
-    }
-
-  }
-
-}
diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdIndexerTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdIndexerTest.java
new file mode 100644
index 00000000000..4b5ff488828
--- /dev/null
+++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdIndexerTest.java
@@ -0,0 +1,106 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.cpd;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.fs.FileSystem;
+import org.sonar.api.batch.fs.internal.DefaultFileSystem;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.config.Settings;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.index.BatchComponentCache;
+import org.sonar.duplications.block.Block;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+
+public class JavaCpdIndexerTest {
+  private static final String JAVA = "java";
+
+  @Mock
+  private SonarDuplicationsIndex index;
+
+  @Captor
+  private ArgumentCaptor<List<Block>> blockCaptor;
+
+  private Settings settings;
+  private JavaCpdIndexer engine;
+  private DefaultInputFile file;
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  @Before
+  public void setUp() throws IOException {
+    MockitoAnnotations.initMocks(this);
+
+    File baseDir = temp.newFolder();
+    DefaultFileSystem fs = new DefaultFileSystem(baseDir);
+    file = new DefaultInputFile("foo", "src/ManyStatements.java").setLanguage(JAVA);
+    fs.add(file);
+    BatchComponentCache batchComponentCache = new BatchComponentCache();
+    batchComponentCache.add(org.sonar.api.resources.File.create("src/Foo.java").setEffectiveKey("foo:src/ManyStatements.java"), null).setInputComponent(file);
+    File ioFile = file.file();
+    FileUtils.copyURLToFile(this.getClass().getResource("ManyStatements.java"), ioFile);
+
+    settings = new Settings();
+    engine = new JavaCpdIndexer(fs, settings, index);
+  }
+
+  @Test
+  public void languageSupported() {
+    JavaCpdIndexer engine = new JavaCpdIndexer(mock(FileSystem.class), new Settings(), index);
+    assertThat(engine.isLanguageSupported(JAVA)).isTrue();
+    assertThat(engine.isLanguageSupported("php")).isFalse();
+  }
+
+  @Test
+  public void testExclusions() {
+    settings.setProperty(CoreProperties.CPD_EXCLUSIONS, "**");
+    engine.index(JAVA);
+    verifyZeroInteractions(index);
+  }
+
+  @Test
+  public void testJavaIndexing() throws Exception {
+    engine.index(JAVA);
+
+    verify(index).insert(eq(file), blockCaptor.capture());
+    List<Block> blockList = blockCaptor.getValue();
+
+    assertThat(blockList).hasSize(26);
+  }
+}
diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java
index bb41a88d7e6..83158d89d1b 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java
@@ -81,11 +81,79 @@ public class CpdMediumTest {
   }
 
   @Test
+  public void testCrossModuleDuplications() throws IOException {
+    builder.put("sonar.modules", "module1,module2")
+      .put("sonar.cpd.xoo.minimumTokens", "10")
+      .put("sonar.verbose", "true");
+
+    // module 1
+    builder.put("module1.sonar.projectKey", "module1");
+    builder.put("module1.sonar.projectName", "Module 1");
+    builder.put("module1.sonar.sources", ".");
+
+    // module2
+    builder.put("module2.sonar.projectKey", "module2");
+    builder.put("module2.sonar.projectName", "Module 2");
+    builder.put("module2.sonar.sources", ".");
+
+    File module1Dir = new File(baseDir, "module1");
+    File module2Dir = new File(baseDir, "module2");
+
+    module1Dir.mkdir();
+    module2Dir.mkdir();
+
+    String duplicatedStuff = "Sample xoo\ncontent\n"
+      + "foo\nbar\ntoto\ntiti\n"
+      + "foo\nbar\ntoto\ntiti\n"
+      + "bar\ntoto\ntiti\n"
+      + "foo\nbar\ntoto\ntiti";
+
+    // create duplicated file in both modules
+    File xooFile1 = new File(module1Dir, "sample1.xoo");
+    FileUtils.write(xooFile1, duplicatedStuff);
+
+    File xooFile2 = new File(module2Dir, "sample2.xoo");
+    FileUtils.write(xooFile2, duplicatedStuff);
+
+    TaskResult result = tester.newTask().properties(builder.build()).start();
+
+    assertThat(result.inputFiles()).hasSize(2);
+
+    InputFile inputFile1 = result.inputFile("sample1.xoo");
+    InputFile inputFile2 = result.inputFile("sample2.xoo");
+
+    // One clone group on each file
+    List<org.sonar.batch.protocol.output.BatchReport.Duplication> duplicationGroupsFile1 = result.duplicationsFor(inputFile1);
+    assertThat(duplicationGroupsFile1).hasSize(1);
+
+    org.sonar.batch.protocol.output.BatchReport.Duplication cloneGroupFile1 = duplicationGroupsFile1.get(0);
+    assertThat(cloneGroupFile1.getOriginPosition().getStartLine()).isEqualTo(1);
+    assertThat(cloneGroupFile1.getOriginPosition().getEndLine()).isEqualTo(17);
+    assertThat(cloneGroupFile1.getDuplicateList()).hasSize(1);
+    assertThat(cloneGroupFile1.getDuplicate(0).getOtherFileRef()).isEqualTo(result.getReportComponent(((DefaultInputFile) inputFile2).key()).getRef());
+
+    List<org.sonar.batch.protocol.output.BatchReport.Duplication> duplicationGroupsFile2 = result.duplicationsFor(inputFile2);
+    assertThat(duplicationGroupsFile2).hasSize(1);
+
+    org.sonar.batch.protocol.output.BatchReport.Duplication cloneGroupFile2 = duplicationGroupsFile2.get(0);
+    assertThat(cloneGroupFile2.getOriginPosition().getStartLine()).isEqualTo(1);
+    assertThat(cloneGroupFile2.getOriginPosition().getEndLine()).isEqualTo(17);
+    assertThat(cloneGroupFile2.getDuplicateList()).hasSize(1);
+    assertThat(cloneGroupFile2.getDuplicate(0).getOtherFileRef()).isEqualTo(result.getReportComponent(((DefaultInputFile) inputFile1).key()).getRef());
+
+    assertThat(result.duplicationBlocksFor(inputFile1)).isEmpty();
+  }
+
+  @Test
   public void testCrossFileDuplications() throws IOException {
     File srcDir = new File(baseDir, "src");
     srcDir.mkdir();
 
-    String duplicatedStuff = "Sample xoo\ncontent\nfoo\nbar\ntoto\ntiti\nfoo\nbar\ntoto\ntiti\nbar\ntoto\ntiti\nfoo\nbar\ntoto\ntiti";
+    String duplicatedStuff = "Sample xoo\ncontent\n"
+      + "foo\nbar\ntoto\ntiti\n"
+      + "foo\nbar\ntoto\ntiti\n"
+      + "bar\ntoto\ntiti\n"
+      + "foo\nbar\ntoto\ntiti";
 
     File xooFile1 = new File(srcDir, "sample1.xoo");
     FileUtils.write(xooFile1, duplicatedStuff);
@@ -103,8 +171,6 @@ public class CpdMediumTest {
 
     assertThat(result.inputFiles()).hasSize(2);
 
-    Map<String, List<org.sonar.batch.protocol.output.BatchReport.Measure>> allMeasures = result.allMeasures();
-
     InputFile inputFile1 = result.inputFile("src/sample1.xoo");
     InputFile inputFile2 = result.inputFile("src/sample2.xoo");