SONAR-6000 Try to decrease size of duplications in persistit

plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java

@@ -26,7 +26,8 @@ import org.sonar.xoo.lang.MeasureSensor;
 import org.sonar.xoo.lang.SymbolReferencesSensor;
 import org.sonar.xoo.lang.SyntaxHighlightingSensor;
 import org.sonar.xoo.lang.TestCaseSensor;
-import org.sonar.xoo.lang.XooTokenizerSensor;
+import org.sonar.xoo.lang.XooCpdMapping;
+import org.sonar.xoo.lang.XooTokenizer;
 import org.sonar.xoo.rule.ChecksSensor;
 import org.sonar.xoo.rule.CreateIssueByInternalKeySensor;
 import org.sonar.xoo.rule.DeprecatedResourceApiSensor;
@@ -67,11 +68,14 @@ public class XooPlugin extends SonarPlugin {
       XooScmProvider.class,
       XooBlameCommand.class,
 
+      // CPD
+      XooCpdMapping.class,
+      XooTokenizer.class,
+
       // sensors
       MeasureSensor.class,
       SyntaxHighlightingSensor.class,
       SymbolReferencesSensor.class,
-      XooTokenizerSensor.class,
       TestCaseSensor.class,
       CoveragePerTestSensor.class,
       DependencySensor.class,

plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooCpdMapping.java

@@ -0,0 +1,46 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.xoo.lang;
+
+import net.sourceforge.pmd.cpd.Tokenizer;
+import org.sonar.api.batch.AbstractCpdMapping;
+import org.sonar.api.resources.Language;
+import org.sonar.xoo.Xoo;
+
+public class XooCpdMapping extends AbstractCpdMapping {
+
+  private Xoo xoo;
+  private XooTokenizer xooTokenizer;
+
+  public XooCpdMapping(Xoo xoo, XooTokenizer xooTokenizer) {
+    this.xoo = xoo;
+    this.xooTokenizer = xooTokenizer;
+  }
+
+  @Override
+  public Tokenizer getTokenizer() {
+    return xooTokenizer;
+  }
+
+  @Override
+  public Language getLanguage() {
+    return xoo;
+  }
+}

plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java

@@ -0,0 +1,58 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.xoo.lang;
+
+import com.google.common.base.Splitter;
+import net.sourceforge.pmd.cpd.SourceCode;
+import net.sourceforge.pmd.cpd.TokenEntry;
+import net.sourceforge.pmd.cpd.Tokenizer;
+import net.sourceforge.pmd.cpd.Tokens;
+import org.apache.commons.io.FileUtils;
+import org.sonar.api.BatchComponent;
+import org.sonar.api.batch.fs.FileSystem;
+
+import java.io.File;
+import java.io.IOException;
+
+public class XooTokenizer implements Tokenizer, BatchComponent {
+
+  private FileSystem fs;
+
+  public XooTokenizer(FileSystem fs) {
+    this.fs = fs;
+  }
+
+  public final void tokenize(SourceCode source, Tokens cpdTokens) {
+    String fileName = source.getFileName();
+    int lineIdx = 1;
+    try {
+      for (String line : FileUtils.readLines(new File(fileName), fs.encoding())) {
+        for (String token : Splitter.on(" ").split(line)) {
+          TokenEntry cpdToken = new TokenEntry(token, fileName, lineIdx);
+          cpdTokens.add(cpdToken);
+        }
+        lineIdx++;
+      }
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to tokenize", e);
+    }
+    cpdTokens.add(TokenEntry.getEOF());
+  }
+}

plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizerSensor.java

@@ -1,84 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.xoo.lang;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.FilePredicates;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.sensor.Sensor;
-import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.batch.sensor.SensorDescriptor;
-import org.sonar.api.batch.sensor.duplication.DuplicationTokenBuilder;
-import org.sonar.xoo.Xoo;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Tokenize xoo files (separator is whitespace) for duplication detection
- */
-public class XooTokenizerSensor implements Sensor {
-
-  private void computeTokens(InputFile inputFile, SensorContext context) {
-    DuplicationTokenBuilder tokenBuilder = context.duplicationTokenBuilder(inputFile);
-    File ioFile = inputFile.file();
-    int lineId = 0;
-    try {
-      for (String line : FileUtils.readLines(ioFile)) {
-        lineId++;
-        for (String token : Splitter.on(" ").split(line)) {
-          tokenBuilder.addToken(lineId, token);
-        }
-      }
-      tokenBuilder.done();
-    } catch (IOException e) {
-      throw new IllegalStateException("Unable to read file " + ioFile, e);
-    }
-  }
-
-  @Override
-  public void describe(SensorDescriptor descriptor) {
-    descriptor
-      .name("Xoo Tokenizer Sensor")
-      .onlyOnLanguages(Xoo.KEY)
-      .onlyOnFileType(InputFile.Type.MAIN);
-  }
-
-  @Override
-  public void execute(SensorContext context) {
-    String[] cpdExclusions = context.settings().getStringArray(CoreProperties.CPD_EXCLUSIONS);
-    FilePredicates p = context.fileSystem().predicates();
-    List<InputFile> sourceFiles = Lists.newArrayList(context.fileSystem().inputFiles(p.and(
-      p.hasType(InputFile.Type.MAIN),
-      p.hasLanguage(Xoo.KEY),
-      p.doesNotMatchPathPatterns(cpdExclusions)
-      )));
-    if (sourceFiles.isEmpty()) {
-      return;
-    }
-    for (InputFile file : sourceFiles) {
-      computeTokens(file, context);
-    }
-  }
-}

plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/XooPluginTest.java

@@ -27,6 +27,6 @@ public class XooPluginTest {
 
   @Test
   public void provide_extensions() {
-    assertThat(new XooPlugin().getExtensions()).hasSize(21);
+    assertThat(new XooPlugin().getExtensions().size()).isGreaterThan(0);
   }
 }

plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerSensorTest.java → plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerTest.java

@@ -19,32 +19,28 @@
  */
 package org.sonar.xoo.lang;
 
+import net.sourceforge.pmd.cpd.SourceCode;
+import net.sourceforge.pmd.cpd.TokenEntry;
+import net.sourceforge.pmd.cpd.Tokens;
 import org.apache.commons.io.FileUtils;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.batch.sensor.duplication.DuplicationTokenBuilder;
-import org.sonar.api.batch.sensor.internal.DefaultSensorDescriptor;
 import org.sonar.api.config.Settings;
 
 import java.io.File;
 import java.io.IOException;
 
-import static org.mockito.Matchers.any;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-public class XooTokenizerSensorTest {
+public class XooTokenizerTest {
 
-  private XooTokenizerSensor sensor;
   private SensorContext context = mock(SensorContext.class);
   private DefaultFileSystem fileSystem;
 
@@ -56,42 +52,27 @@ public class XooTokenizerSensorTest {
   @Before
   public void prepare() throws IOException {
     baseDir = temp.newFolder();
-    sensor = new XooTokenizerSensor();
     fileSystem = new DefaultFileSystem(baseDir.toPath());
     when(context.fileSystem()).thenReturn(fileSystem);
     settings = new Settings();
     when(context.settings()).thenReturn(settings);
   }
 
-  @Test
-  public void testDescriptor() {
-    sensor.describe(new DefaultSensorDescriptor());
-  }
-
-  @Test
-  public void testNoExecutionIfExclusion() {
-    DefaultInputFile inputFile = new DefaultInputFile("foo", "src/foo.xoo").setLanguage("xoo");
-    fileSystem.add(inputFile);
-    settings.setProperty(CoreProperties.CPD_EXCLUSIONS, "**/foo.xoo");
-    sensor.execute(context);
-    verify(context, never()).duplicationTokenBuilder(any(InputFile.class));
-  }
-
   @Test
   public void testExecution() throws IOException {
     File source = new File(baseDir, "src/foo.xoo");
     FileUtils.write(source, "token1 token2 token3\ntoken4");
     DefaultInputFile inputFile = new DefaultInputFile("foo", "src/foo.xoo").setLanguage("xoo");
     fileSystem.add(inputFile);
-    DuplicationTokenBuilder builder = mock(DuplicationTokenBuilder.class);
-    when(context.duplicationTokenBuilder(inputFile)).thenReturn(builder);
 
-    sensor.execute(context);
+    XooTokenizer tokenizer = new XooTokenizer(fileSystem);
+    SourceCode sourceCode = mock(SourceCode.class);
+    when(sourceCode.getFileName()).thenReturn(inputFile.absolutePath());
+    Tokens cpdTokens = new Tokens();
+    tokenizer.tokenize(sourceCode, cpdTokens);
 
-    verify(builder).addToken(1, "token1");
-    verify(builder).addToken(1, "token2");
-    verify(builder).addToken(1, "token3");
-    verify(builder).addToken(2, "token4");
-    verify(builder).done();
+    // 4 tokens + EOF
+    assertThat(cpdTokens.getTokens()).hasSize(5);
+    assertThat(cpdTokens.getTokens().get(3)).isEqualTo(new TokenEntry("token4", "src/foo.xoo", 2));
   }
 }

sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java

@@ -19,9 +19,6 @@
  */
 package org.sonar.batch.cpd;
 
-import org.sonar.batch.cpd.index.IndexFactory;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
-
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterables;
@@ -38,10 +35,10 @@ import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Project;
 import org.sonar.api.utils.SonarException;
-import org.sonar.batch.duplication.BlockCache;
+import org.sonar.batch.cpd.index.IndexFactory;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
 import org.sonar.duplications.DuplicationPredicates;
 import org.sonar.duplications.block.Block;
-import org.sonar.duplications.block.FileBlocks;
 import org.sonar.duplications.index.CloneGroup;
 import org.sonar.duplications.internal.pmd.TokenizerBridge;
 
@@ -68,20 +65,18 @@ public class DefaultCpdEngine extends CpdEngine {
   private final CpdMappings mappings;
   private final FileSystem fs;
   private final Settings settings;
-  private final BlockCache blockCache;
   private final Project project;
 
-  public DefaultCpdEngine(@Nullable Project project, IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings, BlockCache duplicationCache) {
+  public DefaultCpdEngine(@Nullable Project project, IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings) {
     this.project = project;
     this.indexFactory = indexFactory;
     this.mappings = mappings;
     this.fs = fs;
     this.settings = settings;
-    this.blockCache = duplicationCache;
   }
 
-  public DefaultCpdEngine(IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings, BlockCache duplicationCache) {
-    this(null, indexFactory, mappings, fs, settings, duplicationCache);
+  public DefaultCpdEngine(IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings) {
+    this(null, indexFactory, mappings, fs, settings);
   }
 
   @Override
@@ -91,6 +86,12 @@ public class DefaultCpdEngine extends CpdEngine {
 
   @Override
   public void analyse(String languageKey, SensorContext context) {
+    CpdMapping mapping = mappings.getMapping(languageKey);
+    if (mapping == null) {
+      LOG.debug("No CpdMapping for language " + languageKey);
+      return;
+    }
+
     String[] cpdExclusions = settings.getStringArray(CoreProperties.CPD_EXCLUSIONS);
     logExclusions(cpdExclusions, LOG);
     FilePredicates p = fs.predicates();
@@ -103,8 +104,6 @@ public class DefaultCpdEngine extends CpdEngine {
       return;
     }
 
-    CpdMapping mapping = mappings.getMapping(languageKey);
-
     // Create index
     SonarDuplicationsIndex index = indexFactory.create(project, languageKey);
     populateIndex(languageKey, sourceFiles, mapping, index);
@@ -140,20 +139,12 @@ public class DefaultCpdEngine extends CpdEngine {
   }
 
   private void populateIndex(String languageKey, List<InputFile> sourceFiles, CpdMapping mapping, SonarDuplicationsIndex index) {
-    TokenizerBridge bridge = null;
-    if (mapping != null) {
-      bridge = new TokenizerBridge(mapping.getTokenizer(), fs.encoding().name(), getBlockSize(languageKey));
-    }
+    TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), fs.encoding().name(), getBlockSize(languageKey));
     for (InputFile inputFile : sourceFiles) {
       LOG.debug("Populating index from {}", inputFile);
       String resourceEffectiveKey = ((DeprecatedDefaultInputFile) inputFile).key();
-      FileBlocks fileBlocks = blockCache.byComponent(resourceEffectiveKey);
-      if (fileBlocks != null) {
-        index.insert(inputFile, fileBlocks.blocks());
-      } else if (bridge != null) {
-        List<Block> blocks2 = bridge.chunk(resourceEffectiveKey, inputFile.file());
-        index.insert(inputFile, blocks2);
-      }
+      List<Block> blocks2 = bridge.chunk(resourceEffectiveKey, inputFile.file());
+      index.insert(inputFile, blocks2);
     }
   }
 

sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java

@@ -20,9 +20,6 @@
 
 package org.sonar.batch.cpd;
 
-import org.sonar.batch.cpd.index.IndexFactory;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
-
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import org.apache.commons.io.IOUtils;
@@ -34,14 +31,16 @@ import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
 import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.batch.sensor.duplication.DuplicationBuilder;
-import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplicationBuilder;
+import org.sonar.api.batch.sensor.duplication.NewDuplication;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure;
 import org.sonar.api.config.Settings;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.resources.Project;
 import org.sonar.api.utils.KeyValueFormat;
 import org.sonar.api.utils.SonarException;
+import org.sonar.batch.cpd.index.IndexFactory;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.block.BlockChunker;
 import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm;
@@ -66,7 +65,12 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 public class JavaCpdEngine extends CpdEngine {
 
@@ -226,16 +230,17 @@ public class JavaCpdEngine extends CpdEngine {
       .setFromCore()
       .save();
 
-    DuplicationBuilder builder = context.duplicationBuilder(inputFile);
     for (CloneGroup duplication : duplications) {
-      builder.originBlock(duplication.getOriginPart().getStartLine(), duplication.getOriginPart().getEndLine());
+      NewDuplication builder = context.newDuplication();
+      ClonePart originPart = duplication.getOriginPart();
+      builder.originBlock(inputFile, originPart.getStartLine(), originPart.getEndLine());
       for (ClonePart part : duplication.getCloneParts()) {
-        if (!part.equals(duplication.getOriginPart())) {
-          ((DefaultDuplicationBuilder) builder).isDuplicatedBy(part.getResourceId(), part.getStartLine(), part.getEndLine());
+        if (!part.equals(originPart)) {
+          ((DefaultDuplication) builder).isDuplicatedBy(part.getResourceId(), part.getStartLine(), part.getEndLine());
         }
       }
+      builder.save();
     }
-    context.saveDuplications(inputFile, builder.build());
   }
 
   private static int computeBlockAndLineCount(Iterable<CloneGroup> duplications, Set<Integer> duplicatedLines) {

sonar-batch/src/main/java/org/sonar/batch/deprecated/DeprecatedSensorContext.java

@@ -45,7 +45,6 @@ import org.sonar.api.resources.Qualifiers;
 import org.sonar.api.resources.Resource;
 import org.sonar.api.rules.Violation;
 import org.sonar.api.utils.SonarException;
-import org.sonar.batch.duplication.BlockCache;
 import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.batch.index.ComponentDataCache;
 import org.sonar.batch.sensor.DefaultSensorContext;
@@ -67,8 +66,8 @@ public class DeprecatedSensorContext extends DefaultSensorContext implements Sen
 
   public DeprecatedSensorContext(SonarIndex index, Project project, Settings settings, FileSystem fs, ActiveRules activeRules,
     AnalysisMode analysisMode, ComponentDataCache componentDataCache, CoverageExclusions coverageFilter,
-    BlockCache blockCache, DuplicationCache duplicationCache, SensorStorage sensorStorage) {
-    super(settings, fs, activeRules, analysisMode, componentDataCache, blockCache, duplicationCache, sensorStorage);
+    DuplicationCache duplicationCache, SensorStorage sensorStorage) {
+    super(settings, fs, activeRules, analysisMode, componentDataCache, duplicationCache, sensorStorage);
     this.index = index;
     this.project = project;
     this.coverageFilter = coverageFilter;

sonar-batch/src/main/java/org/sonar/batch/deprecated/decorator/DefaultDecoratorContext.java

@@ -25,7 +25,7 @@ import com.google.common.collect.Lists;
 import org.sonar.api.batch.DecoratorContext;
 import org.sonar.api.batch.Event;
 import org.sonar.api.batch.SonarIndex;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.design.Dependency;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Measure;
@@ -126,7 +126,7 @@ public class DefaultDecoratorContext implements DecoratorContext {
     String metricKey = ((MeasuresFilters.MetricFilter<M>) filter).filterOnMetricKey();
     if (CoreMetrics.DUPLICATIONS_DATA_KEY.equals(metricKey)) {
       // Hack for SONAR-5765
-      List<DuplicationGroup> group = duplicationCache.byComponent(resource.getEffectiveKey());
+      Iterable<DefaultDuplication> group = duplicationCache.byComponent(resource.getEffectiveKey());
       if (group != null) {
         unfiltered = Arrays.asList(new Measure(CoreMetrics.DUPLICATIONS_DATA, DuplicationUtils.toXml(group)));
       } else {

sonar-batch/src/main/java/org/sonar/batch/duplication/BlockCache.java

@@ -1,56 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.duplication;
-
-import org.sonar.api.BatchComponent;
-import org.sonar.batch.index.Cache;
-import org.sonar.batch.index.Cache.Entry;
-import org.sonar.batch.index.Caches;
-import org.sonar.duplications.block.FileBlocks;
-
-import javax.annotation.CheckForNull;
-
-/**
- * Cache of duplication blocks. This cache is shared amongst all project modules.
- */
-public class BlockCache implements BatchComponent {
-
-  private final Cache<FileBlocks> cache;
-
-  public BlockCache(Caches caches) {
-    caches.registerValueCoder(FileBlocks.class, new FileBlocksValueCoder());
-    cache = caches.createCache("blocks");
-  }
-
-  public Iterable<Entry<FileBlocks>> entries() {
-    return cache.entries();
-  }
-
-  @CheckForNull
-  public FileBlocks byComponent(String effectiveKey) {
-    return cache.get(effectiveKey);
-  }
-
-  public BlockCache put(String effectiveKey, FileBlocks blocks) {
-    cache.put(effectiveKey, blocks);
-    return this;
-  }
-
-}

sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationGroupValueCoder.java → sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultDuplicationValueCoder.java

@@ -22,35 +22,32 @@ package org.sonar.batch.duplication;
 import com.persistit.Value;
 import com.persistit.encoding.CoderContext;
 import com.persistit.encoding.ValueCoder;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup.Block;
+import org.sonar.api.batch.sensor.duplication.Duplication;
+import org.sonar.api.batch.sensor.duplication.Duplication.Block;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 
-import java.util.ArrayList;
-import java.util.List;
-
-class DuplicationGroupValueCoder implements ValueCoder {
+class DefaultDuplicationValueCoder implements ValueCoder {
 
   private DuplicationBlockValueCoder blockCoder = new DuplicationBlockValueCoder();
 
   @Override
   public void put(Value value, Object object, CoderContext context) {
-    DuplicationGroup c = (DuplicationGroup) object;
+    DefaultDuplication c = (DefaultDuplication) object;
     blockCoder.put(value, c.originBlock(), context);
     value.put(c.duplicates().size());
-    for (DuplicationGroup.Block block : c.duplicates()) {
+    for (Duplication.Block block : c.duplicates()) {
       blockCoder.put(value, block, context);
     }
   }
 
   @Override
   public Object get(Value value, Class clazz, CoderContext context) {
-    DuplicationGroup g = new DuplicationGroup((Block) blockCoder.get(value, DuplicationGroup.Block.class, context));
+    DefaultDuplication g = new DefaultDuplication();
+    g.setOriginBlock((Block) blockCoder.get(value, Duplication.Block.class, context));
     int count = value.getInt();
-    List<DuplicationGroup.Block> blocks = new ArrayList<DuplicationGroup.Block>(count);
     for (int i = 0; i < count; i++) {
-      blocks.add((Block) blockCoder.get(value, DuplicationGroup.Block.class, context));
+      g.duplicates().add((Block) blockCoder.get(value, Duplication.Block.class, context));
     }
-    g.setDuplicates(blocks);
     return g;
   }
 }

sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultTokenBuilder.java

@@ -1,73 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.batch.duplication;
-
-import com.google.common.base.Preconditions;
-import net.sourceforge.pmd.cpd.TokenEntry;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.duplication.DuplicationTokenBuilder;
-import org.sonar.duplications.block.Block;
-import org.sonar.duplications.block.FileBlocks;
-import org.sonar.duplications.internal.pmd.PmdBlockChunker;
-import org.sonar.duplications.internal.pmd.TokenizerBridge;
-import org.sonar.duplications.internal.pmd.TokensLine;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DefaultTokenBuilder implements DuplicationTokenBuilder {
-
-  private final BlockCache cache;
-  private final InputFile inputFile;
-  private final List<TokenEntry> tokens = new ArrayList<TokenEntry>();
-  private final PmdBlockChunker blockChunker;
-  private boolean done = false;
-  private int previousLine = 0;
-
-  public DefaultTokenBuilder(InputFile inputFile, BlockCache cache, PmdBlockChunker blockChunker) {
-    this.inputFile = inputFile;
-    this.cache = cache;
-    this.blockChunker = blockChunker;
-    TokenEntry.clearImages();
-  }
-
-  @Override
-  public DefaultTokenBuilder addToken(int line, String image) {
-    Preconditions.checkState(!done, "done() already called");
-    Preconditions.checkState(line >= previousLine, "Token should be created in order. Previous line was " + previousLine + " and you tried to create a token at line " + line);
-    TokenEntry cpdToken = new TokenEntry(image, inputFile.absolutePath(), line);
-    tokens.add(cpdToken);
-    previousLine = line;
-    return this;
-  }
-
-  @Override
-  public void done() {
-    Preconditions.checkState(!done, "done() already called");
-    tokens.add(TokenEntry.getEOF());
-    TokenEntry.clearImages();
-    List<TokensLine> tokensLines = TokenizerBridge.convert(tokens);
-    List<Block> blocks = blockChunker.chunk(((DefaultInputFile) inputFile).key(), tokensLines);
-
-    cache.put(((DefaultInputFile) inputFile).key(), new FileBlocks(((DefaultInputFile) inputFile).key(), blocks));
-    tokens.clear();
-  }
-}

sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationBlockValueCoder.java

@@ -22,13 +22,13 @@ package org.sonar.batch.duplication;
 import com.persistit.Value;
 import com.persistit.encoding.CoderContext;
 import com.persistit.encoding.ValueCoder;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.Duplication;
 
 class DuplicationBlockValueCoder implements ValueCoder {
 
   @Override
   public void put(Value value, Object object, CoderContext context) {
-    DuplicationGroup.Block b = (DuplicationGroup.Block) object;
+    Duplication.Block b = (Duplication.Block) object;
     value.putUTF(b.resourceKey());
     value.put(b.startLine());
     value.put(b.length());
@@ -39,6 +39,6 @@ class DuplicationBlockValueCoder implements ValueCoder {
     String resourceKey = value.getString();
     int startLine = value.getInt();
     int length = value.getInt();
-    return new DuplicationGroup.Block(resourceKey, startLine, length);
+    return new Duplication.Block(resourceKey, startLine, length);
   }
 }

sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationCache.java

@@ -19,39 +19,41 @@
  */
 package org.sonar.batch.duplication;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
 import org.sonar.api.BatchComponent;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.batch.index.Cache;
-import org.sonar.batch.index.Cache.Entry;
 import org.sonar.batch.index.Caches;
 
-import javax.annotation.CheckForNull;
-
-import java.util.List;
-
 /**
  * Cache of duplication blocks. This cache is shared amongst all project modules.
  */
 public class DuplicationCache implements BatchComponent {
 
-  private final Cache<List<DuplicationGroup>> cache;
+  private final Cache<DefaultDuplication> cache;
+  private int sequence = 1;
 
   public DuplicationCache(Caches caches) {
-    caches.registerValueCoder(DuplicationGroup.class, new DuplicationGroupValueCoder());
+    caches.registerValueCoder(DefaultDuplication.class, new DefaultDuplicationValueCoder());
     cache = caches.createCache("duplications");
   }
 
-  public Iterable<Entry<List<DuplicationGroup>>> entries() {
-    return cache.entries();
+  public Iterable<String> componentKeys() {
+    return Iterables.transform(cache.keySet(), new Function<Object, String>() {
+      @Override
+      public String apply(Object input) {
+        return input.toString();
+      }
+    });
   }
 
-  @CheckForNull
-  public List<DuplicationGroup> byComponent(String effectiveKey) {
-    return cache.get(effectiveKey);
+  public Iterable<DefaultDuplication> byComponent(String effectiveKey) {
+    return cache.values(effectiveKey);
   }
 
-  public DuplicationCache put(String effectiveKey, List<DuplicationGroup> blocks) {
-    cache.put(effectiveKey, blocks);
+  public DuplicationCache put(String effectiveKey, DefaultDuplication duplication) {
+    cache.put(effectiveKey, sequence++, duplication);
     return this;
   }
 

sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationUtils.java

@@ -20,7 +20,8 @@
 package org.sonar.batch.duplication;
 
 import org.apache.commons.lang.StringEscapeUtils;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.Duplication;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 
 public class DuplicationUtils {
 
@@ -28,13 +29,13 @@ public class DuplicationUtils {
     // Utility class
   }
 
-  public static String toXml(Iterable<DuplicationGroup> duplications) {
+  public static String toXml(Iterable<DefaultDuplication> duplications) {
     StringBuilder xml = new StringBuilder();
     xml.append("<duplications>");
-    for (DuplicationGroup duplication : duplications) {
+    for (Duplication duplication : duplications) {
       xml.append("<g>");
       toXml(xml, duplication.originBlock());
-      for (DuplicationGroup.Block part : duplication.duplicates()) {
+      for (Duplication.Block part : duplication.duplicates()) {
         toXml(xml, part);
       }
       xml.append("</g>");
@@ -43,7 +44,7 @@ public class DuplicationUtils {
     return xml.toString();
   }
 
-  private static void toXml(StringBuilder xml, DuplicationGroup.Block part) {
+  private static void toXml(StringBuilder xml, Duplication.Block part) {
     xml.append("<b s=\"").append(part.startLine())
       .append("\" l=\"").append(part.length())
       .append("\" r=\"").append(StringEscapeUtils.escapeXml(part.resourceKey()))

sonar-batch/src/main/java/org/sonar/batch/index/DuplicationPersister.java

@@ -19,7 +19,7 @@
  */
 package org.sonar.batch.index;
 
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.database.model.MeasureMapper;
 import org.sonar.api.database.model.MeasureModel;
 import org.sonar.api.measures.CoreMetrics;
@@ -30,12 +30,9 @@ import org.sonar.api.measures.PersistenceMode;
 import org.sonar.api.rules.RuleFinder;
 import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.batch.duplication.DuplicationUtils;
-import org.sonar.batch.index.Cache.Entry;
 import org.sonar.core.persistence.DbSession;
 import org.sonar.core.persistence.MyBatis;
 
-import java.util.List;
-
 public final class DuplicationPersister implements ScanPersister {
   private final MyBatis mybatis;
   private final RuleFinder ruleFinder;
@@ -58,9 +55,9 @@ public final class DuplicationPersister implements ScanPersister {
     try (DbSession session = mybatis.openSession(false)) {
       MeasureMapper mapper = session.getMapper(MeasureMapper.class);
       Metric duplicationMetricWithId = metricFinder.findByKey(CoreMetrics.DUPLICATIONS_DATA_KEY);
-      for (Entry<List<DuplicationGroup>> entry : duplicationCache.entries()) {
-        String effectiveKey = entry.key()[0].toString();
-        Measure measure = new Measure(duplicationMetricWithId, DuplicationUtils.toXml(entry.value())).setPersistenceMode(PersistenceMode.DATABASE);
+      for (String effectiveKey : duplicationCache.componentKeys()) {
+        Iterable<DefaultDuplication> dups = duplicationCache.byComponent(effectiveKey);
+        Measure measure = new Measure(duplicationMetricWithId, DuplicationUtils.toXml(dups)).setPersistenceMode(PersistenceMode.DATABASE);
         BatchResource batchResource = resourceCache.get(effectiveKey);
 
         if (MeasurePersister.shouldPersistMeasure(batchResource.resource(), measure)) {

sonar-batch/src/main/java/org/sonar/batch/index/SourceDataFactory.java

@@ -24,7 +24,8 @@ import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.BatchComponent;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.Duplication;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.batch.sensor.symbol.Symbol;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Measure;
@@ -341,29 +342,23 @@ public class SourceDataFactory implements BatchComponent {
   }
 
   void applyDuplications(String inputFileKey, FileSourceDb.Data.Builder to) {
-    List<DuplicationGroup> groups = duplicationCache.byComponent(inputFileKey);
-    if (groups != null) {
-      int blockId = 1;
-      for (Iterator<DuplicationGroup> it = groups.iterator(); it.hasNext();) {
-        DuplicationGroup group = it.next();
-        addBlock(blockId, group.originBlock(), to);
-        blockId++;
-        for (Iterator<DuplicationGroup.Block> dupsIt = group.duplicates().iterator(); dupsIt.hasNext();) {
-          DuplicationGroup.Block dups = dupsIt.next();
-          if (inputFileKey.equals(dups.resourceKey())) {
-            addBlock(blockId, dups, to);
-            blockId++;
-          }
-          // Save memory
-          dupsIt.remove();
+    Iterable<DefaultDuplication> groups = duplicationCache.byComponent(inputFileKey);
+    int blockId = 1;
+    for (Iterator<DefaultDuplication> it = groups.iterator(); it.hasNext();) {
+      Duplication group = it.next();
+      addBlock(blockId, group.originBlock(), to);
+      blockId++;
+      for (Iterator<Duplication.Block> dupsIt = group.duplicates().iterator(); dupsIt.hasNext();) {
+        Duplication.Block dups = dupsIt.next();
+        if (inputFileKey.equals(dups.resourceKey())) {
+          addBlock(blockId, dups, to);
+          blockId++;
         }
-        // Save memory
-        it.remove();
       }
     }
   }
 
-  private void addBlock(int blockId, DuplicationGroup.Block block, FileSourceDb.Data.Builder to) {
+  private void addBlock(int blockId, Duplication.Block block, FileSourceDb.Data.Builder to) {
     int currentLine = block.startLine();
     for (int i = 0; i < block.length(); i++) {
       if (currentLine <= to.getLinesCount()) {

sonar-batch/src/main/java/org/sonar/batch/mediumtest/TaskResult.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.batch.mediumtest;
 
+import com.google.common.collect.Lists;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,7 +27,7 @@ import org.sonar.api.batch.fs.InputDir;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.sensor.dependency.Dependency;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.Duplication;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure;
 import org.sonar.api.batch.sensor.symbol.Symbol;
@@ -67,7 +68,7 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver {
 
   private List<Issue> issues = new ArrayList<>();
   private List<org.sonar.api.batch.sensor.measure.Measure> measures = new ArrayList<>();
-  private Map<String, List<DuplicationGroup>> duplications = new HashMap<>();
+  private Map<String, List<Duplication>> duplications = new HashMap<>();
   private Map<String, InputFile> inputFiles = new HashMap<>();
   private Map<String, InputDir> inputDirs = new HashMap<>();
   private Map<InputFile, SyntaxHighlightingData> highlightingPerFile = new HashMap<>();
@@ -142,9 +143,8 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver {
 
   private void storeDuplication(ProjectScanContainer container) {
     DuplicationCache duplicationCache = container.getComponentByType(DuplicationCache.class);
-    for (Entry<List<DuplicationGroup>> entry : duplicationCache.entries()) {
-      String effectiveKey = entry.key()[0].toString();
-      duplications.put(effectiveKey, entry.value());
+    for (String effectiveKey : duplicationCache.componentKeys()) {
+      duplications.put(effectiveKey, Lists.<Duplication>newArrayList(duplicationCache.byComponent(effectiveKey)));
     }
   }
 
@@ -210,7 +210,7 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver {
     return inputDirs.get(relativePath);
   }
 
-  public List<DuplicationGroup> duplicationsFor(InputFile inputFile) {
+  public List<Duplication> duplicationsFor(InputFile inputFile) {
     return duplications.get(((DefaultInputFile) inputFile).key());
   }
 

sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java

@@ -19,10 +19,6 @@
  */
 package org.sonar.batch.scan;
 
-import org.sonar.batch.deprecated.components.DefaultResourceCreationLock;
-
-import org.sonar.batch.deprecated.components.PeriodsDefinition;
-import org.sonar.batch.repository.language.DefaultLanguagesRepository;
 import com.google.common.annotations.VisibleForTesting;
 import org.sonar.api.BatchComponent;
 import org.sonar.api.CoreProperties;
@@ -45,7 +41,8 @@ import org.sonar.batch.bootstrap.ExtensionUtils;
 import org.sonar.batch.bootstrap.MetricProvider;
 import org.sonar.batch.debt.DebtModelProvider;
 import org.sonar.batch.debt.IssueChangelogDebtCalculator;
-import org.sonar.batch.duplication.BlockCache;
+import org.sonar.batch.deprecated.components.DefaultResourceCreationLock;
+import org.sonar.batch.deprecated.components.PeriodsDefinition;
 import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.batch.index.Caches;
 import org.sonar.batch.index.ComponentDataCache;
@@ -70,6 +67,7 @@ import org.sonar.batch.phases.GraphPersister;
 import org.sonar.batch.profiling.PhasesSumUpTimeProfiler;
 import org.sonar.batch.repository.ProjectRepositoriesProvider;
 import org.sonar.batch.repository.ProjectScmRepositoryLoader;
+import org.sonar.batch.repository.language.DefaultLanguagesRepository;
 import org.sonar.batch.rule.ActiveRulesProvider;
 import org.sonar.batch.rule.RulesProvider;
 import org.sonar.batch.scan.filesystem.InputPathCache;
@@ -189,7 +187,6 @@ public class ProjectScanContainer extends ComponentContainer {
       MeasureCache.class,
 
       // Duplications
-      BlockCache.class,
       DuplicationCache.class,
 
       ProjectSettings.class,

sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java

@@ -19,7 +19,6 @@
  */
 package org.sonar.batch.sensor;
 
-import com.google.common.base.Preconditions;
 import org.sonar.api.batch.AnalysisMode;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
@@ -29,10 +28,8 @@ import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.batch.sensor.SensorStorage;
 import org.sonar.api.batch.sensor.dependency.Dependency;
 import org.sonar.api.batch.sensor.dependency.internal.DefaultDependency;
-import org.sonar.api.batch.sensor.duplication.DuplicationBuilder;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
-import org.sonar.api.batch.sensor.duplication.DuplicationTokenBuilder;
-import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplicationBuilder;
+import org.sonar.api.batch.sensor.duplication.NewDuplication;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.batch.sensor.highlighting.HighlightingBuilder;
 import org.sonar.api.batch.sensor.issue.Issue;
 import org.sonar.api.batch.sensor.issue.internal.DefaultIssue;
@@ -46,16 +43,12 @@ import org.sonar.api.batch.sensor.test.internal.DefaultCoverage;
 import org.sonar.api.batch.sensor.test.internal.DefaultTestCaseCoverage;
 import org.sonar.api.batch.sensor.test.internal.DefaultTestCaseExecution;
 import org.sonar.api.config.Settings;
-import org.sonar.batch.duplication.BlockCache;
-import org.sonar.batch.duplication.DefaultTokenBuilder;
 import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.batch.highlighting.DefaultHighlightingBuilder;
 import org.sonar.batch.index.ComponentDataCache;
 import org.sonar.batch.symbol.DefaultSymbolTableBuilder;
-import org.sonar.duplications.internal.pmd.PmdBlockChunker;
 
 import java.io.Serializable;
-import java.util.List;
 
 public class DefaultSensorContext implements SensorContext {
 
@@ -63,19 +56,17 @@ public class DefaultSensorContext implements SensorContext {
   private final FileSystem fs;
   private final ActiveRules activeRules;
   private final ComponentDataCache componentDataCache;
-  private final BlockCache blockCache;
   private final DuplicationCache duplicationCache;
   private final SensorStorage sensorStorage;
   private final AnalysisMode analysisMode;
 
   public DefaultSensorContext(Settings settings, FileSystem fs, ActiveRules activeRules, AnalysisMode analysisMode, ComponentDataCache componentDataCache,
-    BlockCache blockCache, DuplicationCache duplicationCache, SensorStorage sensorStorage) {
+    DuplicationCache duplicationCache, SensorStorage sensorStorage) {
     this.settings = settings;
     this.fs = fs;
     this.activeRules = activeRules;
     this.analysisMode = analysisMode;
     this.componentDataCache = componentDataCache;
-    this.blockCache = blockCache;
     this.duplicationCache = duplicationCache;
     this.sensorStorage = sensorStorage;
   }
@@ -121,43 +112,8 @@ public class DefaultSensorContext implements SensorContext {
   }
 
   @Override
-  public DuplicationTokenBuilder duplicationTokenBuilder(InputFile inputFile) {
-    PmdBlockChunker blockChunker = new PmdBlockChunker(getBlockSize(inputFile.language()));
-
-    return new DefaultTokenBuilder(inputFile, blockCache, blockChunker);
-  }
-
-  @Override
-  public DuplicationBuilder duplicationBuilder(InputFile inputFile) {
-    return new DefaultDuplicationBuilder(inputFile);
-  }
-
-  @Override
-  public void saveDuplications(InputFile inputFile, List<DuplicationGroup> duplications) {
-    Preconditions.checkState(!duplications.isEmpty(), "Empty duplications");
-    String effectiveKey = ((DefaultInputFile) inputFile).key();
-    for (DuplicationGroup duplicationGroup : duplications) {
-      Preconditions.checkState(effectiveKey.equals(duplicationGroup.originBlock().resourceKey()), "Invalid duplication group");
-    }
-    duplicationCache.put(effectiveKey, duplications);
-  }
-
-  private int getBlockSize(String languageKey) {
-    int blockSize = settings.getInt("sonar.cpd." + languageKey + ".minimumLines");
-    if (blockSize == 0) {
-      blockSize = getDefaultBlockSize(languageKey);
-    }
-    return blockSize;
-  }
-
-  private static int getDefaultBlockSize(String languageKey) {
-    if ("cobol".equals(languageKey)) {
-      return 30;
-    } else if ("abap".equals(languageKey) || "natur".equals(languageKey)) {
-      return 20;
-    } else {
-      return 10;
-    }
+  public NewDuplication newDuplication() {
+    return new DefaultDuplication(sensorStorage);
   }
 
   @Override

sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java

@@ -26,6 +26,8 @@ import org.sonar.api.batch.fs.InputPath;
 import org.sonar.api.batch.measure.MetricFinder;
 import org.sonar.api.batch.rule.ActiveRules;
 import org.sonar.api.batch.sensor.SensorStorage;
+import org.sonar.api.batch.sensor.duplication.Duplication;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.batch.sensor.issue.Issue;
 import org.sonar.api.batch.sensor.issue.Issue.Severity;
 import org.sonar.api.batch.sensor.measure.Measure;
@@ -53,7 +55,6 @@ import org.sonar.api.test.MutableTestCase;
 import org.sonar.api.test.MutableTestPlan;
 import org.sonar.api.test.MutableTestable;
 import org.sonar.api.test.Testable;
-import org.sonar.batch.duplication.BlockCache;
 import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.batch.index.ComponentDataCache;
 import org.sonar.batch.index.DefaultIndex;
@@ -68,16 +69,18 @@ public class DefaultSensorStorage implements SensorStorage {
   private final ResourcePerspectives perspectives;
   private final DefaultIndex sonarIndex;
   private final CoverageExclusions coverageExclusions;
+  private final DuplicationCache duplicationCache;
 
   public DefaultSensorStorage(MetricFinder metricFinder, Project project,
     ResourcePerspectives perspectives,
-    Settings settings, FileSystem fs, ActiveRules activeRules, ComponentDataCache componentDataCache, BlockCache blockCache,
+    Settings settings, FileSystem fs, ActiveRules activeRules, ComponentDataCache componentDataCache,
     DuplicationCache duplicationCache, DefaultIndex sonarIndex, CoverageExclusions coverageExclusions) {
     this.metricFinder = metricFinder;
     this.project = project;
     this.perspectives = perspectives;
     this.sonarIndex = sonarIndex;
     this.coverageExclusions = coverageExclusions;
+    this.duplicationCache = duplicationCache;
   }
 
   private Metric findMetricOrFail(String metricKey) {
@@ -260,4 +263,9 @@ public class DefaultSensorStorage implements SensorStorage {
       .setWeight(dep.weight())
       .setParent(parentDep));
   }
+
+  @Override
+  public void store(Duplication duplication) {
+    duplicationCache.put(duplication.originBlock().resourceKey(), (DefaultDuplication) duplication);
+  }
 }

sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java

@@ -19,13 +19,6 @@
  */
 package org.sonar.batch.cpd;
 
-import org.sonar.batch.cpd.CpdMappings;
-import org.sonar.batch.cpd.CpdComponents;
-import org.sonar.batch.cpd.CpdSensor;
-import org.sonar.batch.cpd.DefaultCpdEngine;
-import org.sonar.batch.cpd.JavaCpdEngine;
-import org.sonar.batch.cpd.index.IndexFactory;
-
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -34,7 +27,7 @@ import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.config.PropertyDefinitions;
 import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Java;
-import org.sonar.batch.duplication.BlockCache;
+import org.sonar.batch.cpd.index.IndexFactory;
 
 import java.io.IOException;
 
@@ -55,7 +48,7 @@ public class CpdSensorTest {
   public void setUp() throws IOException {
     IndexFactory indexFactory = mock(IndexFactory.class);
     sonarEngine = new JavaCpdEngine(indexFactory, null, null);
-    sonarBridgeEngine = new DefaultCpdEngine(indexFactory, new CpdMappings(), null, null, mock(BlockCache.class));
+    sonarBridgeEngine = new DefaultCpdEngine(indexFactory, new CpdMappings(), null, null);
     settings = new Settings(new PropertyDefinitions(CpdComponents.class));
 
     DefaultFileSystem fs = new DefaultFileSystem(temp.newFolder().toPath());

sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java

@@ -19,14 +19,12 @@
  */
 package org.sonar.batch.cpd;
 
-import org.sonar.batch.cpd.DefaultCpdEngine;
-
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Project;
-import org.sonar.batch.duplication.BlockCache;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.mock;
@@ -42,7 +40,7 @@ public class DefaultCpdEngineTest {
   @Before
   public void init() {
     settings = new Settings();
-    engine = new DefaultCpdEngine(null, null, null, settings, mock(BlockCache.class));
+    engine = new DefaultCpdEngine(null, null, null, settings);
   }
 
   @Test

sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java

@@ -19,22 +19,17 @@
  */
 package org.sonar.batch.cpd;
 
-import org.sonar.batch.cpd.JavaCpdEngine;
-
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.mockito.InOrder;
-import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
-import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
 import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.batch.sensor.SensorStorage;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
-import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplicationBuilder;
+import org.sonar.api.batch.sensor.duplication.NewDuplication;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.batch.sensor.measure.Measure;
 import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure;
 import org.sonar.api.measures.CoreMetrics;
@@ -46,9 +41,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
@@ -60,7 +53,6 @@ public class JavaCpdEngineTest {
 
   SensorContext context = mock(SensorContext.class);
   DeprecatedDefaultInputFile inputFile;
-  private DefaultDuplicationBuilder duplicationBuilder;
   private SensorStorage storage = mock(SensorStorage.class);
 
   @Before
@@ -71,9 +63,13 @@ public class JavaCpdEngineTest {
         return new DefaultMeasure(storage);
       }
     });
-    inputFile = new DeprecatedDefaultInputFile("foo", "src/main/java/Foo.java");
-    duplicationBuilder = spy(new DefaultDuplicationBuilder(inputFile));
-    when(context.duplicationBuilder(any(InputFile.class))).thenReturn(duplicationBuilder);
+    when(context.newDuplication()).then(new Answer<NewDuplication>() {
+      @Override
+      public NewDuplication answer(InvocationOnMock invocation) throws Throwable {
+        return new DefaultDuplication(storage);
+      }
+    });
+    inputFile = (DeprecatedDefaultInputFile) new DeprecatedDefaultInputFile("foo", "src/main/java/Foo.java").setLines(300);
     inputFile.setModuleBaseDir(temp.newFolder().toPath());
   }
 
@@ -97,10 +93,9 @@ public class JavaCpdEngineTest {
     verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_LINES).onFile(inputFile).withValue(3));
     verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATION_LINES_DATA).onFile(inputFile).withValue("1=0;2=1;3=1;4=1;5=0"));
 
-    InOrder inOrder = Mockito.inOrder(duplicationBuilder);
-    inOrder.verify(duplicationBuilder).originBlock(2, 4);
-    inOrder.verify(duplicationBuilder).isDuplicatedBy("key2", 15, 17);
-    inOrder.verify(duplicationBuilder).build();
+    verify(storage).store(new DefaultDuplication()
+      .originBlock(inputFile, 2, 4)
+      .isDuplicatedBy("key2", 15, 17));
   }
 
   @Test
@@ -112,10 +107,9 @@ public class JavaCpdEngineTest {
     verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_BLOCKS).onFile(inputFile).withValue(2));
     verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_LINES).onFile(inputFile).withValue(400));
 
-    InOrder inOrder = Mockito.inOrder(duplicationBuilder);
-    inOrder.verify(duplicationBuilder).originBlock(5, 204);
-    inOrder.verify(duplicationBuilder).isDuplicatedBy("key1", 215, 414);
-    inOrder.verify(duplicationBuilder).build();
+    verify(storage).store(new DefaultDuplication()
+      .originBlock(inputFile, 5, 204)
+      .isDuplicatedBy("key1", 215, 414));
   }
 
   @Test
@@ -127,17 +121,10 @@ public class JavaCpdEngineTest {
     verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_BLOCKS).onFile(inputFile).withValue(1));
     verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_LINES).onFile(inputFile).withValue(200));
 
-    InOrder inOrder = Mockito.inOrder(duplicationBuilder);
-    inOrder.verify(duplicationBuilder).originBlock(5, 204);
-    inOrder.verify(duplicationBuilder).isDuplicatedBy("key2", 15, 214);
-    inOrder.verify(duplicationBuilder).isDuplicatedBy("key3", 25, 224);
-    inOrder.verify(duplicationBuilder).build();
-
-    verify(context).saveDuplications(inputFile, Arrays.asList(
-      new DuplicationGroup(new DuplicationGroup.Block("foo:src/main/java/Foo.java", 5, 200))
-        .addDuplicate(new DuplicationGroup.Block("key2", 15, 200))
-        .addDuplicate(new DuplicationGroup.Block("key3", 25, 200))
-      ));
+    verify(storage).store(new DefaultDuplication()
+      .originBlock(inputFile, 5, 204)
+      .isDuplicatedBy("key2", 15, 214)
+      .isDuplicatedBy("key3", 25, 224));
   }
 
   @Test
@@ -151,12 +138,12 @@ public class JavaCpdEngineTest {
     verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_BLOCKS).onFile(inputFile).withValue(2));
     verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_LINES).onFile(inputFile).withValue(210));
 
-    InOrder inOrder = Mockito.inOrder(duplicationBuilder);
-    inOrder.verify(duplicationBuilder).originBlock(5, 204);
-    inOrder.verify(duplicationBuilder).isDuplicatedBy("key2", 15, 214);
-    inOrder.verify(duplicationBuilder).originBlock(15, 214);
-    inOrder.verify(duplicationBuilder).isDuplicatedBy("key3", 15, 214);
-    inOrder.verify(duplicationBuilder).build();
+    verify(storage).store(new DefaultDuplication()
+      .originBlock(inputFile, 5, 204)
+      .isDuplicatedBy("key2", 15, 214));
+    verify(storage).store(new DefaultDuplication()
+      .originBlock(inputFile, 15, 214)
+      .isDuplicatedBy("key3", 15, 214));
   }
 
   private CloneGroup newCloneGroup(ClonePart... parts) {

sonar-batch/src/test/java/org/sonar/batch/duplication/DuplicationCacheTest.java

@@ -25,14 +25,11 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.Duplication;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.batch.index.Caches;
 import org.sonar.batch.index.CachesTest;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class DuplicationCacheTest {
@@ -60,24 +57,28 @@ public class DuplicationCacheTest {
   public void should_add_clone_groups() throws Exception {
     DuplicationCache cache = new DuplicationCache(caches);
 
-    DuplicationGroup group1 = new DuplicationGroup(new DuplicationGroup.Block("foo", 1, 2))
-      .addDuplicate(new DuplicationGroup.Block("foo", 1, 2))
-      .addDuplicate(new DuplicationGroup.Block("foo2", 12, 22))
-      .addDuplicate(new DuplicationGroup.Block("foo3", 13, 23));
+    DefaultDuplication group1 = new DefaultDuplication()
+      .setOriginBlock(new Duplication.Block("foo", 1, 2));
+    group1.duplicates().add(new Duplication.Block("foo", 1, 2));
+    group1.duplicates().add(new Duplication.Block("foo2", 12, 22));
+    group1.duplicates().add(new Duplication.Block("foo3", 13, 23));
 
-    DuplicationGroup group2 = new DuplicationGroup(new DuplicationGroup.Block("2foo", 1, 2))
-      .addDuplicate(new DuplicationGroup.Block("2foo", 1, 2))
-      .addDuplicate(new DuplicationGroup.Block("2foo2", 12, 22))
-      .addDuplicate(new DuplicationGroup.Block("2foo3", 13, 23));
+    DefaultDuplication group2 = new DefaultDuplication()
+      .setOriginBlock(new Duplication.Block("2foo", 1, 2));
+    group2.duplicates().add(new Duplication.Block("2foo", 1, 2));
+    group2.duplicates().add(new Duplication.Block("2foo2", 12, 22));
+    group2.duplicates().add(new Duplication.Block("2foo3", 13, 23));
 
-    assertThat(cache.entries()).hasSize(0);
+    assertThat(cache.componentKeys()).hasSize(0);
 
-    cache.put("foo", new ArrayList<DuplicationGroup>(Arrays.asList(group1, group2)));
+    cache.put("foo", group1);
+    cache.put("foo", group2);
 
-    assertThat(cache.entries()).hasSize(1);
+    assertThat(cache.componentKeys()).hasSize(1);
+    assertThat(cache.byComponent("foo")).hasSize(2);
 
-    List<DuplicationGroup> entry = cache.byComponent("foo");
-    assertThat(entry.get(0).originBlock().resourceKey()).isEqualTo("foo");
+    Iterable<DefaultDuplication> entry = cache.byComponent("foo");
+    assertThat(entry.iterator().next().originBlock().resourceKey()).isEqualTo("foo");
 
   }
 

sonar-batch/src/test/java/org/sonar/batch/index/DuplicationPersisterTest.java

@@ -22,7 +22,8 @@ package org.sonar.batch.index;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.Duplication;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.database.model.Snapshot;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.MetricFinder;
@@ -32,7 +33,6 @@ import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.core.persistence.AbstractDaoTestCase;
 
 import java.util.Arrays;
-import java.util.List;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -71,13 +71,14 @@ public class DuplicationPersisterTest extends AbstractDaoTestCase {
   public void should_insert_duplications() {
     setupData("empty");
 
-    DuplicationGroup.Block originBlock = new DuplicationGroup.Block("foo:org/foo/Bar.java", 1, 4);
+    Duplication.Block originBlock = new Duplication.Block("foo:org/foo/Bar.java", 1, 4);
 
-    DuplicationGroup group = new DuplicationGroup(originBlock)
-      .addDuplicate(new DuplicationGroup.Block("foo:org/foo/Foo.java", 5, 9));
+    DefaultDuplication group = new DefaultDuplication().setOriginBlock(originBlock);
+    group.duplicates().add(new Duplication.Block("foo:org/foo/Foo.java", 5, 9));
 
-    when(duplicationCache.entries()).thenReturn(
-      Arrays.<Cache.Entry<List<DuplicationGroup>>>asList(new Cache.Entry(new String[] {"foo:org/foo/Bar.java"}, Arrays.asList(group))));
+    when(duplicationCache.componentKeys()).thenReturn(Arrays.asList("foo:org/foo/Bar.java"));
+
+    when(duplicationCache.byComponent("foo:org/foo/Bar.java")).thenReturn(Arrays.asList(group));
 
     duplicationPersister.persist();
 

sonar-batch/src/test/java/org/sonar/batch/index/SourceDataFactoryTest.java

@@ -28,7 +28,8 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.Duplication;
+import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Measure;
@@ -45,8 +46,10 @@ import org.sonar.server.source.db.FileSourceDb;
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.Collections;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -76,6 +79,7 @@ public class SourceDataFactoryTest {
     metadata = new InputFileMetadata();
     FileUtils.write(inputFile.file(), "one\ntwo\nthree\n");
     output = sut.createForSource(inputFile);
+    when(duplicationCache.byComponent(anyString())).thenReturn(Collections.<DefaultDuplication>emptyList());
   }
 
   @Test
@@ -170,13 +174,13 @@ public class SourceDataFactoryTest {
 
   @Test
   public void applyDuplications() throws Exception {
-    DuplicationGroup group1 = new DuplicationGroup(new DuplicationGroup.Block(inputFile.key(), 1, 1))
-      .addDuplicate(new DuplicationGroup.Block(inputFile.key(), 3, 1))
-      .addDuplicate(new DuplicationGroup.Block("anotherFile1", 12, 1))
-      .addDuplicate(new DuplicationGroup.Block("anotherFile2", 13, 1));
-    DuplicationGroup group2 = new DuplicationGroup(new DuplicationGroup.Block(inputFile.key(), 1, 2))
-      .addDuplicate(new DuplicationGroup.Block("anotherFile1", 12, 2))
-      .addDuplicate(new DuplicationGroup.Block("anotherFile2", 13, 2));
+    DefaultDuplication group1 = new DefaultDuplication().setOriginBlock(new Duplication.Block(inputFile.key(), 1, 1));
+    group1.duplicates().add(new Duplication.Block(inputFile.key(), 3, 1));
+    group1.duplicates().add(new Duplication.Block("anotherFile1", 12, 1));
+    group1.duplicates().add(new Duplication.Block("anotherFile2", 13, 1));
+    DefaultDuplication group2 = new DefaultDuplication().setOriginBlock(new Duplication.Block(inputFile.key(), 1, 2));
+    group2.duplicates().add(new Duplication.Block("anotherFile1", 12, 2));
+    group2.duplicates().add(new Duplication.Block("anotherFile2", 13, 2));
     when(duplicationCache.byComponent(inputFile.key())).thenReturn(Lists.newArrayList(group1, group2));
 
     sut.applyDuplications(inputFile.key(), output);
@@ -190,9 +194,9 @@ public class SourceDataFactoryTest {
   @Test
   public void applyDuplications_ignore_bad_lines() throws Exception {
     // duplication on 10 lines
-    DuplicationGroup group1 = new DuplicationGroup(new DuplicationGroup.Block(inputFile.key(), 1, 10))
-      .addDuplicate(new DuplicationGroup.Block("anotherFile1", 12, 1))
-      .addDuplicate(new DuplicationGroup.Block("anotherFile2", 13, 1));
+    DefaultDuplication group1 = new DefaultDuplication().setOriginBlock(new Duplication.Block(inputFile.key(), 1, 10));
+    group1.duplicates().add(new Duplication.Block("anotherFile1", 12, 1));
+    group1.duplicates().add(new Duplication.Block("anotherFile2", 13, 1));
     when(duplicationCache.byComponent(inputFile.key())).thenReturn(Lists.newArrayList(group1));
 
     sut.applyDuplications(inputFile.key(), output);

sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java

@@ -29,7 +29,7 @@ import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
+import org.sonar.api.batch.sensor.duplication.Duplication;
 import org.sonar.batch.mediumtest.BatchMediumTester;
 import org.sonar.batch.mediumtest.TaskResult;
 import org.sonar.xoo.XooPlugin;
@@ -106,10 +106,10 @@ public class CpdMediumTest {
     InputFile inputFile1 = result.inputFile("src/sample1.xoo");
     InputFile inputFile2 = result.inputFile("src/sample2.xoo");
     // One clone group on each file
-    List<DuplicationGroup> duplicationGroupsFile1 = result.duplicationsFor(inputFile1);
+    List<Duplication> duplicationGroupsFile1 = result.duplicationsFor(inputFile1);
     assertThat(duplicationGroupsFile1).hasSize(1);
 
-    DuplicationGroup cloneGroupFile1 = duplicationGroupsFile1.get(0);
+    Duplication cloneGroupFile1 = duplicationGroupsFile1.get(0);
     assertThat(cloneGroupFile1.duplicates()).hasSize(1);
     assertThat(cloneGroupFile1.originBlock().startLine()).isEqualTo(1);
     assertThat(cloneGroupFile1.originBlock().length()).isEqualTo(17);
@@ -117,10 +117,10 @@ public class CpdMediumTest {
     assertThat(cloneGroupFile1.duplicates()).hasSize(1);
     assertThat(cloneGroupFile1.duplicates().get(0).resourceKey()).isEqualTo(((DefaultInputFile) inputFile2).key());
 
-    List<DuplicationGroup> duplicationGroupsFile2 = result.duplicationsFor(inputFile2);
+    List<Duplication> duplicationGroupsFile2 = result.duplicationsFor(inputFile2);
     assertThat(duplicationGroupsFile2).hasSize(1);
 
-    DuplicationGroup cloneGroupFile2 = duplicationGroupsFile2.get(0);
+    Duplication cloneGroupFile2 = duplicationGroupsFile2.get(0);
     assertThat(cloneGroupFile2.duplicates()).hasSize(1);
     assertThat(cloneGroupFile2.originBlock().startLine()).isEqualTo(1);
     assertThat(cloneGroupFile2.originBlock().length()).isEqualTo(17);
@@ -153,10 +153,10 @@ public class CpdMediumTest {
 
     InputFile inputFile = result.inputFile("src/sample.xoo");
     // One clone group
-    List<DuplicationGroup> duplicationGroups = result.duplicationsFor(inputFile);
+    List<Duplication> duplicationGroups = result.duplicationsFor(inputFile);
     assertThat(duplicationGroups).hasSize(1);
 
-    DuplicationGroup cloneGroup = duplicationGroups.get(0);
+    Duplication cloneGroup = duplicationGroups.get(0);
     assertThat(cloneGroup.duplicates()).hasSize(1);
     assertThat(cloneGroup.originBlock().startLine()).isEqualTo(1);
     assertThat(cloneGroup.originBlock().length()).isEqualTo(2);

sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorContextTest.java

@@ -32,7 +32,6 @@ import org.sonar.api.batch.rule.internal.ActiveRulesBuilder;
 import org.sonar.api.batch.sensor.SensorStorage;
 import org.sonar.api.config.Settings;
 import org.sonar.api.measures.CoreMetrics;
-import org.sonar.batch.duplication.BlockCache;
 import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.batch.index.ComponentDataCache;
 
@@ -64,10 +63,9 @@ public class DefaultSensorContextTest {
     when(metricFinder.findByKey(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY)).thenReturn(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION);
     settings = new Settings();
     ComponentDataCache componentDataCache = mock(ComponentDataCache.class);
-    BlockCache blockCache = mock(BlockCache.class);
     sensorStorage = mock(SensorStorage.class);
     analysisMode = mock(AnalysisMode.class);
-    adaptor = new DefaultSensorContext(settings, fs, activeRules, analysisMode, componentDataCache, blockCache, mock(DuplicationCache.class), sensorStorage);
+    adaptor = new DefaultSensorContext(settings, fs, activeRules, analysisMode, componentDataCache, mock(DuplicationCache.class), sensorStorage);
   }
 
   @Test

sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java

@@ -51,7 +51,6 @@ import org.sonar.api.resources.File;
 import org.sonar.api.resources.Project;
 import org.sonar.api.resources.Resource;
 import org.sonar.api.rule.RuleKey;
-import org.sonar.batch.duplication.BlockCache;
 import org.sonar.batch.duplication.DuplicationCache;
 import org.sonar.batch.index.ComponentDataCache;
 import org.sonar.batch.index.DefaultIndex;
@@ -91,13 +90,12 @@ public class DefaultSensorStorageTest {
     settings = new Settings();
     resourcePerspectives = mock(ResourcePerspectives.class);
     ComponentDataCache componentDataCache = mock(ComponentDataCache.class);
-    BlockCache blockCache = mock(BlockCache.class);
     project = new Project("myProject");
     sonarIndex = mock(DefaultIndex.class);
     CoverageExclusions coverageExclusions = mock(CoverageExclusions.class);
     when(coverageExclusions.accept(any(Resource.class), any(Measure.class))).thenReturn(true);
     sensorStorage = new DefaultSensorStorage(metricFinder, project,
-      resourcePerspectives, settings, fs, activeRules, componentDataCache, blockCache, mock(DuplicationCache.class), sonarIndex, coverageExclusions);
+      resourcePerspectives, settings, fs, activeRules, componentDataCache, mock(DuplicationCache.class), sonarIndex, coverageExclusions);
   }
 
   @Test

sonar-duplications/pom.xml

@@ -30,6 +30,10 @@
       <artifactId>jsr305</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>commons-lang</groupId>
+      <artifactId>commons-lang</artifactId>
+    </dependency>
 
     <!-- unit tests -->
     <dependency>

sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java

@@ -24,6 +24,7 @@
 package net.sourceforge.pmd.cpd;
 
 import com.google.common.annotations.Beta;
+import org.apache.commons.lang.builder.ToStringBuilder;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -136,4 +137,9 @@ public class TokenEntry implements Comparable<TokenEntry> {
   public int compareTo(TokenEntry other) {
     return getIndex() - other.getIndex();
   }
+
+  @Override
+  public String toString() {
+    return ToStringBuilder.reflectionToString(this);
+  }
 }

sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java

@@ -26,10 +26,29 @@ package net.sourceforge.pmd.cpd;
 import java.io.IOException;
 
 /**
+ * A tokenizer is responsible to return a token list for the provided input file (see {@link SourceCode#getFileName()}.
+ * Tokens are basically list of non empty words in a file but you can also do some "anonymization" to ignore litteral differences.
+ * 
+ * For example if you have a first file:
+ * <pre>
+ * public class MyClass1 {
+ *   int foo1;
+ * }
+ * </pre>
+ * and a second file:
+ * <pre>
+ * public class MyClass2 {
+ *   int foo2;
+ * }
+ * </pre>
+ * Then in both cases your tokenizer could return the following (line, image) list:
+ * <pre>(1,public),(1,class),(1,LITERAL),(1,{),(2,int),(2,LITERAL),(2,;),(3,})</pre>
+ * in this case the two files will be considered as duplicate.
+ * 
  * @since 2.2
  */
 public interface Tokenizer {
 
-  void tokenize(SourceCode tokens, Tokens tokenEntries) throws IOException;
+  void tokenize(SourceCode sourceFile, Tokens tokenEntries) throws IOException;
 
 }

sonar-plugin-api/src/main/java/org/sonar/api/batch/CpdMapping.java

@@ -28,6 +28,7 @@ import java.io.File;
 import java.util.List;
 
 /**
+ * Implement this extension to get Copy/Paste detection for your language.
  * @since 1.10
  */
 public interface CpdMapping extends BatchExtension {

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorContext.java

@@ -20,13 +20,12 @@
 package org.sonar.api.batch.sensor;
 
 import org.sonar.api.batch.AnalysisMode;
+import org.sonar.api.batch.CpdMapping;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.rule.ActiveRules;
 import org.sonar.api.batch.sensor.dependency.Dependency;
-import org.sonar.api.batch.sensor.duplication.DuplicationBuilder;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
-import org.sonar.api.batch.sensor.duplication.DuplicationTokenBuilder;
+import org.sonar.api.batch.sensor.duplication.NewDuplication;
 import org.sonar.api.batch.sensor.highlighting.HighlightingBuilder;
 import org.sonar.api.batch.sensor.issue.Issue;
 import org.sonar.api.batch.sensor.measure.Measure;
@@ -38,7 +37,6 @@ import org.sonar.api.batch.sensor.test.TestCaseExecution;
 import org.sonar.api.config.Settings;
 
 import java.io.Serializable;
-import java.util.List;
 
 /**
  * See {@link Sensor#execute(SensorContext)}
@@ -99,24 +97,11 @@ public interface SensorContext {
   // ------------ DUPLICATIONS ------------
 
   /**
-   * Builder to define tokens in a file. Tokens are used to compute duplication using default SonarQube engine.
-   * @since 4.5
-   */
-  DuplicationTokenBuilder duplicationTokenBuilder(InputFile inputFile);
-
-  /**
-   * Builder to manually define duplications in a file. When duplication are manually computed then
-   * no need to use {@link #duplicationTokenBuilder(InputFile)}.
-   * @since 4.5
-   */
-  DuplicationBuilder duplicationBuilder(InputFile inputFile);
-
-  /**
-   * Register all duplications of an {@link InputFile}. Use {@link #duplicationBuilder(InputFile)} to create
-   * list of duplications.
-   * @since 4.5
+   * Builder to manually register duplication in a file. This can be used in addition to {@link CpdMapping} extension point.
+   * Don't forget to call {@link NewDuplication#save()}.
+   * @since 5.1
    */
-  void saveDuplications(InputFile inputFile, List<DuplicationGroup> duplications);
+  NewDuplication newDuplication();
 
   // ------------ TESTS ------------
 

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorStorage.java

@@ -20,6 +20,7 @@
 package org.sonar.api.batch.sensor;
 
 import org.sonar.api.batch.sensor.dependency.Dependency;
+import org.sonar.api.batch.sensor.duplication.Duplication;
 import org.sonar.api.batch.sensor.issue.Issue;
 import org.sonar.api.batch.sensor.measure.Measure;
 import org.sonar.api.batch.sensor.test.TestCaseCoverage;
@@ -27,7 +28,7 @@ import org.sonar.api.batch.sensor.test.TestCaseExecution;
 
 /**
  * Interface for storing data computed by sensors.
- * @since 5.0
+ * @since 5.1
  */
 public interface SensorStorage {
 
@@ -35,6 +36,8 @@ public interface SensorStorage {
 
   void store(Issue issue);
 
+  void store(Duplication duplication);
+
   void store(TestCaseExecution testCaseExecution);
 
   void store(Dependency dependency);

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/duplication/DuplicationGroup.java → sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/duplication/Duplication.java

@@ -19,27 +19,23 @@
  */
 package org.sonar.api.batch.sensor.duplication;
 
-import com.google.common.annotations.Beta;
 import org.apache.commons.lang.builder.EqualsBuilder;
 import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.commons.lang.builder.ToStringBuilder;
 import org.apache.commons.lang.builder.ToStringStyle;
 import org.sonar.api.batch.sensor.SensorContext;
 
-import java.util.LinkedList;
 import java.util.List;
 
 /**
- * Experimental, do not use.
  * <p/>
- * A {@link DuplicationGroup} is a list of duplicated {@link Block}s. One block
+ * A {@link Duplication} is a list of duplicated {@link Block}s. One block
  * is considered as the original code and all others are duplicates.
- * Use {@link SensorContext#duplicationBuilder(org.sonar.api.batch.fs.InputFile)} and
- * {@link SensorContext#saveDuplications(org.sonar.api.batch.fs.InputFile, List)}.
- * @since 4.5
+ * Use {@link SensorContext#newDuplication()} to manually create a duplication. Use {@link SensorContext#duplicationTokenBuilder(org.sonar.api.batch.fs.InputFile)}
+ * to feed tokens and let the core compute duplications.
+ * @since 5.1
  */
-@Beta
-public class DuplicationGroup {
+public interface Duplication {
 
   public static class Block {
     private final String resourceKey;
@@ -101,80 +97,8 @@ public class DuplicationGroup {
     }
   }
 
-  private final Block originBlock;
-  private List<Block> duplicates = new LinkedList<DuplicationGroup.Block>();
+  public Block originBlock();
 
-  /**
-   * For unit test and internal use only.
-   */
-  public DuplicationGroup(Block originBlock) {
-    this.originBlock = originBlock;
-  }
-
-  /**
-   * For unit test and internal use only.
-   */
-  public void setDuplicates(List<Block> duplicates) {
-    this.duplicates = duplicates;
-  }
-
-  /**
-   * For unit test and internal use only.
-   */
-  public DuplicationGroup addDuplicate(Block anotherBlock) {
-    this.duplicates.add(anotherBlock);
-    return this;
-  }
-
-  public Block originBlock() {
-    return originBlock;
-  }
-
-  public List<Block> duplicates() {
-    return duplicates;
-  }
-
-  // Just for unit tests
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == null) {
-      return false;
-    }
-    if (obj == this) {
-      return true;
-    }
-    if (obj.getClass() != getClass()) {
-      return false;
-    }
-    DuplicationGroup rhs = (DuplicationGroup) obj;
-    EqualsBuilder equalsBuilder = new EqualsBuilder()
-      .append(originBlock, rhs.originBlock)
-      .append(duplicates.size(), rhs.duplicates.size());
-    if (duplicates.size() == rhs.duplicates.size()) {
-      for (int i = 0; i < duplicates.size(); i++) {
-        equalsBuilder.append(duplicates.get(i), rhs.duplicates.get(i));
-      }
-    }
-    return equalsBuilder.isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder hcBuilder = new HashCodeBuilder(17, 37)
-      .append(originBlock)
-      .append(duplicates.size());
-    for (int i = 0; i < duplicates.size(); i++) {
-      hcBuilder.append(duplicates.get(i));
-    }
-    return hcBuilder.toHashCode();
-  }
-
-  @Override
-  public String toString() {
-    return new ToStringBuilder(this, ToStringStyle.MULTI_LINE_STYLE).
-      append("origin", originBlock).
-      append("duplicates", duplicates, true).
-      toString();
-  }
+  public List<Block> duplicates();
 
 }

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/duplication/DuplicationBuilder.java → sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/duplication/NewDuplication.java

@@ -19,45 +19,36 @@
  */
 package org.sonar.api.batch.sensor.duplication;
 
-import com.google.common.annotations.Beta;
 import org.sonar.api.batch.fs.InputFile;
 
-import java.util.List;
-
 /**
- * Experimental, do not use.
  * <p/>
  * This builder is used to declare duplications on files of the project.
  * Usage:
  * <code><pre>
- * DuplicationBuilder builder = context.duplicationBuilder(inputFile);
- *   .originBlock(2, 10)
+ * context.newDuplication();
+ *   .originBlock(inputFile, 2, 10)
  *   .isDuplicatedBy(inputFile, 14, 22)
  *   .isDuplicatedBy(anotherInputFile, 3, 11)
- *   // Start another duplication
- *   .originBlock(45, 50)
- *   .isDuplicatedBy(yetAnotherInputFile, 10, 15);
- *   context.saveDuplications(inputFile, builder.build());
+ *   .save();
  * </pre></code>
- * @since 4.5
+ * @since 5.1
  */
-@Beta
-public interface DuplicationBuilder {
+public interface NewDuplication {
 
   /**
    * Declare duplication origin block. Then call {@link #isDuplicatedBy(InputFile, int, int)} to reference all duplicates of this block.
-   * Then call again {@link #originBlock(int, int)} to declare another duplication.
    */
-  DuplicationBuilder originBlock(int startLine, int endLine);
+  NewDuplication originBlock(InputFile originFile, int startLine, int endLine);
 
   /**
-   * Declare duplicate block of the previously declared {@link #originBlock(int, int)}.
+   * Declare duplicate block of the previously declared {@link #originBlock(int, int)}. Can be called several times.
    * @param sameOrOtherFile duplicate can be in the same file or in another file.
    */
-  DuplicationBuilder isDuplicatedBy(InputFile sameOrOtherFile, int startLine, int endLine);
+  NewDuplication isDuplicatedBy(InputFile sameOrOtherFile, int startLine, int endLine);
 
   /**
-   * Call this method when you have declared all duplications of the file.
+   * Save the duplication.
    */
-  List<DuplicationGroup> build();
+  void save();
 }

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/duplication/internal/DefaultDuplication.java

@@ -0,0 +1,144 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.sensor.duplication.internal;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.sensor.SensorStorage;
+import org.sonar.api.batch.sensor.duplication.Duplication;
+import org.sonar.api.batch.sensor.duplication.NewDuplication;
+import org.sonar.api.batch.sensor.internal.DefaultStorable;
+
+import javax.annotation.Nullable;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class DefaultDuplication extends DefaultStorable implements NewDuplication, Duplication {
+
+  private Block originBlock;
+  private List<Block> duplicates = new LinkedList<Duplication.Block>();
+
+  public DefaultDuplication() {
+    super();
+  }
+
+  public DefaultDuplication(@Nullable SensorStorage storage) {
+    super(storage);
+  }
+
+  @Override
+  public DefaultDuplication originBlock(InputFile inputFile, int startLine, int endLine) {
+    Preconditions.checkArgument(inputFile != null, "InputFile can't be null");
+    validateLineArgument(inputFile, startLine, "startLine");
+    validateLineArgument(inputFile, endLine, "endLine");
+    originBlock = new Duplication.Block(((DefaultInputFile) inputFile).key(), startLine, endLine - startLine + 1);
+    return this;
+  }
+
+  @Override
+  public DefaultDuplication isDuplicatedBy(InputFile sameOrOtherFile, int startLine, int endLine) {
+    Preconditions.checkArgument(sameOrOtherFile != null, "InputFile can't be null");
+    validateLineArgument(sameOrOtherFile, startLine, "startLine");
+    validateLineArgument(sameOrOtherFile, endLine, "endLine");
+    return isDuplicatedBy(((DefaultInputFile) sameOrOtherFile).key(), startLine, endLine);
+  }
+
+  /**
+   * For internal use. Global duplications are referencing files outside of current project so
+   * no way to manipulate an InputFile.
+   */
+  public DefaultDuplication isDuplicatedBy(String fileKey, int startLine, int endLine) {
+    Preconditions.checkNotNull(originBlock, "Call originBlock() first");
+    duplicates.add(new Duplication.Block(fileKey, startLine, endLine - startLine + 1));
+    return this;
+  }
+
+  @Override
+  public void doSave() {
+    Preconditions.checkNotNull(originBlock, "Call originBlock() first");
+    Preconditions.checkState(!duplicates.isEmpty(), "No duplicates. Call isDuplicatedBy()");
+    storage.store(this);
+  }
+
+  @Override
+  public Block originBlock() {
+    return originBlock;
+  }
+
+  public DefaultDuplication setOriginBlock(Block originBlock) {
+    this.originBlock = originBlock;
+    return this;
+  }
+
+  @Override
+  public List<Block> duplicates() {
+    return duplicates;
+  }
+
+  // Just for unit tests
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == null) {
+      return false;
+    }
+    if (obj == this) {
+      return true;
+    }
+    if (obj.getClass() != getClass()) {
+      return false;
+    }
+    DefaultDuplication rhs = (DefaultDuplication) obj;
+    EqualsBuilder equalsBuilder = new EqualsBuilder()
+      .append(originBlock, rhs.originBlock)
+      .append(duplicates.size(), rhs.duplicates.size());
+    if (duplicates.size() == rhs.duplicates.size()) {
+      for (int i = 0; i < duplicates.size(); i++) {
+        equalsBuilder.append(duplicates.get(i), rhs.duplicates.get(i));
+      }
+    }
+    return equalsBuilder.isEquals();
+  }
+
+  @Override
+  public int hashCode() {
+    HashCodeBuilder hcBuilder = new HashCodeBuilder(17, 37)
+      .append(originBlock)
+      .append(duplicates.size());
+    for (int i = 0; i < duplicates.size(); i++) {
+      hcBuilder.append(duplicates.get(i));
+    }
+    return hcBuilder.toHashCode();
+  }
+
+  @Override
+  public String toString() {
+    return new ToStringBuilder(this, ToStringStyle.MULTI_LINE_STYLE).
+      append("origin", originBlock).
+      append("duplicates", duplicates, true).
+      toString();
+  }
+
+}

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/duplication/internal/DefaultDuplicationBuilder.java

@@ -1,79 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.api.batch.sensor.duplication.internal;
-
-import com.google.common.base.Preconditions;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.duplication.DuplicationBuilder;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DefaultDuplicationBuilder implements DuplicationBuilder {
-
-  private final InputFile inputFile;
-  private DuplicationGroup current = null;
-  private List<DuplicationGroup> duplications;
-
-  public DefaultDuplicationBuilder(InputFile inputFile) {
-    this.inputFile = inputFile;
-    duplications = new ArrayList<DuplicationGroup>();
-  }
-
-  @Override
-  public DuplicationBuilder originBlock(int startLine, int endLine) {
-    if (current != null) {
-      duplications.add(current);
-    }
-    current = new DuplicationGroup(new DuplicationGroup.Block(((DefaultInputFile) inputFile).key(), startLine, endLine - startLine + 1));
-    return this;
-  }
-
-  @Override
-  public DuplicationBuilder isDuplicatedBy(InputFile sameOrOtherFile, int startLine, int endLine) {
-    return isDuplicatedBy(((DefaultInputFile) sameOrOtherFile).key(), startLine, endLine);
-  }
-
-  /**
-   * For internal use. Global duplications are referencing files outside of current project so
-   * no way to manipulate an InputFile.
-   */
-  public DuplicationBuilder isDuplicatedBy(String fileKey, int startLine, int endLine) {
-    Preconditions.checkNotNull(current, "Call originBlock() first");
-    current.addDuplicate(new DuplicationGroup.Block(fileKey, startLine, endLine - startLine + 1));
-    return this;
-  }
-
-  @Override
-  public List<DuplicationGroup> build() {
-    Preconditions.checkNotNull(current, "Call originBlock() first");
-    duplications.add(current);
-    List<DuplicationGroup> result = duplications;
-    reset();
-    return result;
-  }
-
-  private void reset() {
-    duplications = new ArrayList<DuplicationGroup>();
-    current = null;
-  }
-}

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/DefaultStorable.java

@@ -22,6 +22,7 @@ package org.sonar.api.batch.sensor.internal;
 import com.google.common.base.Preconditions;
 import org.apache.commons.lang.builder.ToStringBuilder;
 import org.apache.commons.lang.builder.ToStringStyle;
+import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.sensor.SensorStorage;
 
 import javax.annotation.Nullable;
@@ -53,4 +54,9 @@ public abstract class DefaultStorable {
     return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
   }
 
+  protected void validateLineArgument(InputFile inputFile, int line, String label) {
+    Preconditions.checkArgument(line > 0 && line <= inputFile.lines(), "Invalid " + label + ": " + line + ". File " + inputFile + " has " + inputFile.lines()
+      + " lines.");
+  }
+
 }

sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/duplication/DuplicationGroupTest.java

@@ -1,62 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.api.batch.sensor.duplication;
-
-import org.junit.Test;
-
-import java.util.Arrays;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class DuplicationGroupTest {
-
-  @Test
-  public void testBlockEqualsAndCo() {
-    DuplicationGroup.Block b1 = new DuplicationGroup.Block("foo", 1, 10);
-    DuplicationGroup.Block b2 = new DuplicationGroup.Block("foo", 1, 10);
-    assertThat(b1).isEqualTo(b1);
-    assertThat(b1).isEqualTo(b2);
-    assertThat(b1).isNotEqualTo("");
-    assertThat(b1).isNotEqualTo(new DuplicationGroup.Block("foo1", 1, 10));
-    assertThat(b1).isNotEqualTo(new DuplicationGroup.Block("foo", 2, 10));
-    assertThat(b1).isNotEqualTo(new DuplicationGroup.Block("foo", 1, 11));
-
-    assertThat(b1.hashCode()).isEqualTo(188843970);
-    assertThat(b1.toString()).isEqualTo("DuplicationGroup.Block[resourceKey=foo,startLine=1,length=10]");
-  }
-
-  @Test
-  public void testDuplicationGroupEqualsAndCo() {
-    DuplicationGroup d1 = new DuplicationGroup(new DuplicationGroup.Block("foo", 1, 10));
-    d1.setDuplicates(Arrays.asList(new DuplicationGroup.Block("foo", 20, 10), new DuplicationGroup.Block("foo2", 1, 10)));
-    DuplicationGroup d2 = new DuplicationGroup(new DuplicationGroup.Block("foo", 1, 10));
-    d2.setDuplicates(Arrays.asList(new DuplicationGroup.Block("foo", 20, 10), new DuplicationGroup.Block("foo2", 1, 10)));
-    assertThat(d1).isEqualTo(d1);
-    assertThat(d1).isEqualTo(d2);
-    assertThat(d1).isNotEqualTo("");
-    assertThat(d1).isNotEqualTo(new DuplicationGroup(new DuplicationGroup.Block("foo", 1, 10)));
-
-    assertThat(d1.hashCode()).isEqualTo(578909124);
-    assertThat(d1.toString()).contains("origin=DuplicationGroup.Block[resourceKey=foo,startLine=1,length=10]");
-    assertThat(d1.toString()).contains(
-      "duplicates=[DuplicationGroup.Block[resourceKey=foo,startLine=20,length=10], DuplicationGroup.Block[resourceKey=foo2,startLine=1,length=10]]");
-  }
-
-}

sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/duplication/DuplicationTest.java

@@ -0,0 +1,43 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.sensor.duplication;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class DuplicationTest {
+
+  @Test
+  public void testBlockEqualsAndCo() {
+    Duplication.Block b1 = new Duplication.Block("foo", 1, 10);
+    Duplication.Block b2 = new Duplication.Block("foo", 1, 10);
+    assertThat(b1).isEqualTo(b1);
+    assertThat(b1).isEqualTo(b2);
+    assertThat(b1).isNotEqualTo("");
+    assertThat(b1).isNotEqualTo(new Duplication.Block("foo1", 1, 10));
+    assertThat(b1).isNotEqualTo(new Duplication.Block("foo", 2, 10));
+    assertThat(b1).isNotEqualTo(new Duplication.Block("foo", 1, 11));
+
+    assertThat(b1.hashCode()).isEqualTo(188843970);
+    assertThat(b1.toString()).isEqualTo("Duplication.Block[resourceKey=foo,startLine=1,length=10]");
+  }
+
+}

sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/duplication/internal/DefaultDuplicationBuilderTest.java

@@ -1,52 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.api.batch.sensor.duplication.internal;
-
-import org.junit.Test;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
-import org.sonar.api.batch.sensor.duplication.DuplicationGroup.Block;
-
-import java.util.List;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class DefaultDuplicationBuilderTest {
-
-  @Test
-  public void test() {
-    DefaultDuplicationBuilder builder = new DefaultDuplicationBuilder(new DefaultInputFile("foo", "foo.php"));
-
-    List<DuplicationGroup> duplicationGroup = builder.originBlock(1, 11)
-      .isDuplicatedBy(new DefaultInputFile("foo", "foo.php"), 40, 50)
-      .isDuplicatedBy(new DefaultInputFile("foo", "foo2.php"), 1, 10)
-      .originBlock(20, 30)
-      .isDuplicatedBy(new DefaultInputFile("foo", "foo3.php"), 30, 40)
-      .build();
-
-    assertThat(duplicationGroup).hasSize(2);
-    Block originBlock = duplicationGroup.get(0).originBlock();
-    assertThat(originBlock.resourceKey()).isEqualTo("foo:foo.php");
-    assertThat(originBlock.startLine()).isEqualTo(1);
-    assertThat(originBlock.length()).isEqualTo(11);
-    assertThat(duplicationGroup.get(0).duplicates()).hasSize(2);
-  }
-
-}

sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/duplication/internal/DefaultDuplicationTest.java

@@ -0,0 +1,72 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.sensor.duplication.internal;
+
+import org.junit.Test;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.sensor.duplication.Duplication.Block;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class DefaultDuplicationTest {
+
+  @Test
+  public void testDuplicationEqualsAndCo() {
+    DefaultInputFile file1 = new DefaultInputFile("foo", "bar.txt").setLines(50);
+    DefaultInputFile file2 = new DefaultInputFile("foo", "bar2.txt").setLines(50);
+    DefaultDuplication d1 = new DefaultDuplication()
+      .originBlock(file1, 1, 10)
+      .isDuplicatedBy(file1, 20, 29)
+      .isDuplicatedBy(file2, 1, 10);
+    DefaultDuplication d2 = new DefaultDuplication()
+      .originBlock(file1, 1, 10)
+      .isDuplicatedBy(file1, 20, 29)
+      .isDuplicatedBy(file2, 1, 10);
+    DefaultDuplication d3 = new DefaultDuplication()
+      .originBlock(file1, 1, 10);
+    assertThat(d1).isEqualTo(d1);
+    assertThat(d1).isEqualTo(d2);
+    assertThat(d1).isNotEqualTo("");
+    assertThat(d1).isNotEqualTo(d3);
+
+    assertThat(d1.hashCode()).isNotNull();
+    assertThat(d1.toString()).contains("origin=Duplication.Block[resourceKey=foo:bar.txt,startLine=1,length=10]");
+    assertThat(d1.toString()).contains(
+      "duplicates=[Duplication.Block[resourceKey=foo:bar.txt,startLine=20,length=10], Duplication.Block[resourceKey=foo:bar2.txt,startLine=1,length=10]]");
+  }
+
+  @Test
+  public void test() {
+
+    DefaultInputFile file1 = new DefaultInputFile("foo", "foo.php").setLines(50);
+    DefaultInputFile file2 = new DefaultInputFile("foo", "foo2.php").setLines(50);
+
+    DefaultDuplication dup = new DefaultDuplication().originBlock(file1, 1, 11)
+      .isDuplicatedBy(file1, 40, 50)
+      .isDuplicatedBy(file2, 1, 10);
+
+    Block originBlock = dup.originBlock();
+    assertThat(originBlock.resourceKey()).isEqualTo("foo:foo.php");
+    assertThat(originBlock.startLine()).isEqualTo(1);
+    assertThat(originBlock.length()).isEqualTo(11);
+    assertThat(dup.duplicates()).hasSize(2);
+  }
+
+}