SONAR-7389 Add new Sensor API to provide duplication tokens

plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java

@@ -27,6 +27,7 @@ import org.sonar.xoo.coverage.OverallCoverageSensor;
 import org.sonar.xoo.coverage.UtCoverageSensor;
 import org.sonar.xoo.extensions.XooPostJob;
 import org.sonar.xoo.extensions.XooProjectBuilder;
+import org.sonar.xoo.lang.CpdTokenizerSensor;
 import org.sonar.xoo.lang.MeasureSensor;
 import org.sonar.xoo.lang.SymbolReferencesSensor;
 import org.sonar.xoo.lang.SyntaxHighlightingSensor;
@@ -97,6 +98,7 @@ public class XooPlugin extends SonarPlugin {
       ChecksSensor.class,
       RandomAccessSensor.class,
       DeprecatedResourceApiSensor.class,
+      CpdTokenizerSensor.class,
 
       OneBlockerIssuePerFileSensor.class,
       OneIssuePerLineSensor.class,

plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/CpdTokenizerSensor.java

@@ -0,0 +1,88 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.xoo.lang;
+
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import org.sonar.api.batch.fs.FilePredicates;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.InputFile.Type;
+import org.sonar.api.batch.sensor.Sensor;
+import org.sonar.api.batch.sensor.SensorContext;
+import org.sonar.api.batch.sensor.SensorDescriptor;
+import org.sonar.api.batch.sensor.cpd.NewCpdTokens;
+import org.sonar.xoo.Xoo;
+
+/**
+ * Tokenize files for CPD
+ */
+public class CpdTokenizerSensor implements Sensor {
+
+  public static final String ENABLE_PROP = "sonar.xoo.useNewCpdTokenizerApi";
+
+  private void tokenize(InputFile inputFile, SensorContext context) {
+    int lineIdx = 1;
+    NewCpdTokens newCpdTokens = context.newCpdTokens().onFile(inputFile);
+    try {
+      StringBuilder sb = new StringBuilder();
+      for (String line : FileUtils.readLines(inputFile.file(), context.fileSystem().encoding())) {
+        int startOffset = 0;
+        int endOffset = 0;
+        for (int i = 0; i < line.length(); i++) {
+          char c = line.charAt(i);
+          if (Character.isWhitespace(c)) {
+            if (sb.length() > 0) {
+              newCpdTokens.addToken(inputFile.newRange(lineIdx, startOffset, lineIdx, endOffset), sb.toString());
+              sb.setLength(0);
+            }
+            startOffset = endOffset;
+          } else {
+            sb.append(c);
+          }
+          endOffset++;
+        }
+        if (sb.length() > 0) {
+          newCpdTokens.addToken(inputFile.newRange(lineIdx, startOffset, lineIdx, endOffset), sb.toString());
+          sb.setLength(0);
+        }
+        lineIdx++;
+      }
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to tokenize", e);
+    }
+    newCpdTokens.save();
+  }
+
+  @Override
+  public void describe(SensorDescriptor descriptor) {
+    descriptor
+      .name("Xoo Cpd Tokenizer Sensor")
+      .requireProperty(ENABLE_PROP)
+      .onlyOnLanguages(Xoo.KEY);
+  }
+
+  @Override
+  public void execute(SensorContext context) {
+    FilePredicates p = context.fileSystem().predicates();
+    for (InputFile file : context.fileSystem().inputFiles(p.and(p.hasLanguages(Xoo.KEY), p.hasType(Type.MAIN)))) {
+      tokenize(file, context);
+    }
+  }
+}

plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java

@@ -20,6 +20,8 @@
 package org.sonar.xoo.lang;
 
 import com.google.common.base.Splitter;
+import java.io.File;
+import java.io.IOException;
 import net.sourceforge.pmd.cpd.SourceCode;
 import net.sourceforge.pmd.cpd.TokenEntry;
 import net.sourceforge.pmd.cpd.Tokenizer;
@@ -27,13 +29,14 @@ import net.sourceforge.pmd.cpd.Tokens;
 import org.apache.commons.io.FileUtils;
 import org.sonar.api.batch.BatchSide;
 import org.sonar.api.batch.fs.FileSystem;
-
-import java.io.File;
-import java.io.IOException;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 
 @BatchSide
 public class XooTokenizer implements Tokenizer {
 
+  private static final Logger LOG = Loggers.get(XooTokenizer.class);
+
   private FileSystem fs;
 
   public XooTokenizer(FileSystem fs) {
@@ -43,6 +46,7 @@ public class XooTokenizer implements Tokenizer {
   @Override
   public final void tokenize(SourceCode source, Tokens cpdTokens) {
     String fileName = source.getFileName();
+    LOG.info("Using deprecated tokenizer extension point to tokenize {}", fileName);
     int lineIdx = 1;
     try {
       for (String line : FileUtils.readLines(new File(fileName), fs.encoding())) {

sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchComponents.java

@@ -22,6 +22,7 @@ package org.sonar.batch.bootstrap;
 import com.google.common.collect.Lists;
 import java.util.Collection;
 import java.util.List;
+import org.sonar.api.batch.AnalysisMode;
 import org.sonar.batch.cpd.CpdComponents;
 import org.sonar.batch.issue.tracking.ServerIssueFromWs;
 import org.sonar.batch.issue.tracking.TrackedIssue;
@@ -48,27 +49,9 @@ public class BatchComponents {
     // only static stuff
   }
 
-  public static Collection<Object> all() {
+  public static Collection<Object> all(AnalysisMode analysisMode) {
     List<Object> components = Lists.newArrayList(
       DefaultResourceTypes.get(),
-      // SCM
-      ScmConfiguration.class,
-      ScmSensor.class,
-
-      LinesSensor.class,
-      ZeroCoverageSensor.class,
-      CodeColorizerSensor.class,
-
-      // Issues tracking
-      new Tracker<TrackedIssue, ServerIssueFromWs>(),
-
-      // Reports
-      ConsoleReport.class,
-      JSONReport.class,
-      HtmlReport.class,
-      IssuesReportBuilder.class,
-      SourceProvider.class,
-      RuleNameProvider.class,
 
       // Tasks
       Tasks.class,
@@ -77,8 +60,29 @@ public class BatchComponents {
       ScanTask.DEFINITION,
       ScanTask.class);
     components.addAll(CorePropertyDefinitions.all());
-    // CPD
-    components.addAll(CpdComponents.all());
+    if (!analysisMode.isIssues()) {
+      // SCM
+      components.add(ScmConfiguration.class);
+      components.add(ScmSensor.class);
+
+      components.add(LinesSensor.class);
+      components.add(ZeroCoverageSensor.class);
+      components.add(CodeColorizerSensor.class);
+
+      // CPD
+      components.addAll(CpdComponents.all());
+    } else {
+      // Issues tracking
+      components.add(new Tracker<TrackedIssue, ServerIssueFromWs>());
+
+      // Issues report
+      components.add(ConsoleReport.class);
+      components.add(JSONReport.class);
+      components.add(HtmlReport.class);
+      components.add(IssuesReportBuilder.class);
+      components.add(SourceProvider.class);
+      components.add(RuleNameProvider.class);
+    }
     return components;
   }
 }

sonar-batch/src/main/java/org/sonar/batch/bootstrap/ExtensionInstaller.java

@@ -23,6 +23,7 @@ import java.util.List;
 import javax.annotation.Nullable;
 import org.sonar.api.ExtensionProvider;
 import org.sonar.api.SonarPlugin;
+import org.sonar.api.batch.AnalysisMode;
 import org.sonar.core.platform.ComponentContainer;
 import org.sonar.core.platform.PluginInfo;
 import org.sonar.core.platform.PluginRepository;
@@ -30,15 +31,17 @@ import org.sonar.core.platform.PluginRepository;
 public class ExtensionInstaller {
 
   private final PluginRepository pluginRepository;
+  private final AnalysisMode analysisMode;
 
-  public ExtensionInstaller(PluginRepository pluginRepository) {
+  public ExtensionInstaller(PluginRepository pluginRepository, AnalysisMode analysisMode) {
     this.pluginRepository = pluginRepository;
+    this.analysisMode = analysisMode;
   }
 
   public ExtensionInstaller install(ComponentContainer container, ExtensionMatcher matcher) {
 
     // core components
-    for (Object o : BatchComponents.all()) {
+    for (Object o : BatchComponents.all(analysisMode)) {
       doInstall(container, matcher, null, o);
     }
 

sonar-batch/src/main/java/org/sonar/batch/cpd/CpdComponents.java

@@ -31,8 +31,8 @@ public final class CpdComponents {
     return ImmutableList.of(
       CpdSensor.class,
       CpdMappings.class,
-      JavaCpdIndexer.class,
-      DefaultCpdIndexer.class);
+      JavaCpdBlockIndexer.class,
+      DefaultCpdBlockIndexer.class);
   }
 
 }

sonar-batch/src/main/java/org/sonar/batch/cpd/CpdExecutor.java

@@ -26,7 +26,7 @@ import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.config.Settings;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.batch.index.BatchComponent;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.batch.protocol.output.BatchReport;
@@ -64,7 +64,7 @@ public class CpdExecutor {
   static final int MAX_CLONE_GROUP_PER_FILE = 100;
   static final int MAX_CLONE_PART_PER_GROUP = 100;
 
-  private final SonarDuplicationsIndex index;
+  private final SonarCpdBlockIndex index;
   private final ReportPublisher publisher;
   private final BatchComponentCache batchComponentCache;
   private final Settings settings;
@@ -73,7 +73,7 @@ public class CpdExecutor {
   private int count;
   private int total;
 
-  public CpdExecutor(Settings settings, SonarDuplicationsIndex index, ReportPublisher publisher, BatchComponentCache batchComponentCache) {
+  public CpdExecutor(Settings settings, SonarCpdBlockIndex index, ReportPublisher publisher, BatchComponentCache batchComponentCache) {
     this.settings = settings;
     this.index = index;
     this.publisher = publisher;

sonar-batch/src/main/java/org/sonar/batch/cpd/CpdSensor.java

@@ -40,7 +40,7 @@ public class CpdSensor implements Sensor {
   private Settings settings;
   private FileSystem fs;
 
-  public CpdSensor(JavaCpdIndexer sonarEngine, DefaultCpdIndexer sonarBridgeEngine, Settings settings, FileSystem fs) {
+  public CpdSensor(JavaCpdBlockIndexer sonarEngine, DefaultCpdBlockIndexer sonarBridgeEngine, Settings settings, FileSystem fs) {
     this.sonarEngine = sonarEngine;
     this.sonarBridgeEngine = sonarBridgeEngine;
     this.settings = settings;
@@ -49,8 +49,7 @@ public class CpdSensor implements Sensor {
 
   @Override
   public void describe(SensorDescriptor descriptor) {
-    descriptor.name("CPD Sensor")
-      .disabledInIssues();
+    descriptor.name("CPD Sensor");
   }
 
   @VisibleForTesting

sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdIndexer.java → sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdBlockIndexer.java

@@ -31,20 +31,20 @@ import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.config.Settings;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.internal.pmd.TokenizerBridge;
 
-public class DefaultCpdIndexer extends CpdIndexer {
+public class DefaultCpdBlockIndexer extends CpdIndexer {
 
-  private static final Logger LOG = LoggerFactory.getLogger(DefaultCpdIndexer.class);
+  private static final Logger LOG = LoggerFactory.getLogger(DefaultCpdBlockIndexer.class);
 
   private final CpdMappings mappings;
   private final FileSystem fs;
   private final Settings settings;
-  private final SonarDuplicationsIndex index;
+  private final SonarCpdBlockIndex index;
 
-  public DefaultCpdIndexer(CpdMappings mappings, FileSystem fs, Settings settings, SonarDuplicationsIndex index) {
+  public DefaultCpdBlockIndexer(CpdMappings mappings, FileSystem fs, Settings settings, SonarCpdBlockIndex index) {
     this.mappings = mappings;
     this.fs = fs;
     this.settings = settings;
@@ -76,16 +76,18 @@ public class DefaultCpdIndexer extends CpdIndexer {
     }
 
     // Create index
-    populateIndex(languageKey, sourceFiles, mapping, index);
+    populateIndex(languageKey, sourceFiles, mapping);
   }
 
-  private void populateIndex(String languageKey, List<InputFile> sourceFiles, CpdMapping mapping, SonarDuplicationsIndex index) {
+  private void populateIndex(String languageKey, List<InputFile> sourceFiles, CpdMapping mapping) {
     TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), fs.encoding().name(), getBlockSize(languageKey));
     for (InputFile inputFile : sourceFiles) {
-      LOG.debug("Populating index from {}", inputFile);
-      String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
-      List<Block> blocks = bridge.chunk(resourceEffectiveKey, inputFile.file());
-      index.insert(inputFile, blocks);
+      if (!index.isIndexed(inputFile)) {
+        LOG.debug("Populating index from {}", inputFile);
+        String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
+        List<Block> blocks = bridge.chunk(resourceEffectiveKey, inputFile.file());
+        index.insert(inputFile, blocks);
+      }
     }
   }
 
@@ -99,7 +101,7 @@ public class DefaultCpdIndexer extends CpdIndexer {
   }
 
   @VisibleForTesting
-  static int getDefaultBlockSize(String languageKey) {
+  public static int getDefaultBlockSize(String languageKey) {
     if ("cobol".equals(languageKey)) {
       return 30;
     } else if ("abap".equals(languageKey)) {

sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdIndexer.java → sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdBlockIndexer.java

@@ -34,7 +34,7 @@ import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.config.Settings;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.block.BlockChunker;
 import org.sonar.duplications.java.JavaStatementBuilder;
@@ -43,17 +43,17 @@ import org.sonar.duplications.statement.Statement;
 import org.sonar.duplications.statement.StatementChunker;
 import org.sonar.duplications.token.TokenChunker;
 
-public class JavaCpdIndexer extends CpdIndexer {
+public class JavaCpdBlockIndexer extends CpdIndexer {
 
-  private static final Logger LOG = LoggerFactory.getLogger(JavaCpdIndexer.class);
+  private static final Logger LOG = LoggerFactory.getLogger(JavaCpdBlockIndexer.class);
 
   private static final int BLOCK_SIZE = 10;
 
   private final FileSystem fs;
   private final Settings settings;
-  private final SonarDuplicationsIndex index;
+  private final SonarCpdBlockIndex index;
 
-  public JavaCpdIndexer(FileSystem fs, Settings settings, SonarDuplicationsIndex index) {
+  public JavaCpdBlockIndexer(FileSystem fs, Settings settings, SonarCpdBlockIndex index) {
     this.fs = fs;
     this.settings = settings;
     this.index = index;

sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarDuplicationsIndex.java → sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarCpdBlockIndex.java

@@ -22,8 +22,9 @@ package org.sonar.batch.cpd.index;
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Iterator;
-
+import java.util.Set;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.fs.InputFile;
@@ -38,14 +39,16 @@ import org.sonar.duplications.index.CloneIndex;
 import org.sonar.duplications.index.PackedMemoryCloneIndex;
 import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks;
 
-public class SonarDuplicationsIndex extends AbstractCloneIndex {
+public class SonarCpdBlockIndex extends AbstractCloneIndex {
 
   private final CloneIndex mem = new PackedMemoryCloneIndex();
   private final ReportPublisher publisher;
   private final BatchComponentCache batchComponentCache;
   private final Settings settings;
+  // Files already tokenized
+  private final Set<InputFile> indexedFiles = new HashSet<>();
 
-  public SonarDuplicationsIndex(ReportPublisher publisher, BatchComponentCache batchComponentCache, Settings settings) {
+  public SonarCpdBlockIndex(ReportPublisher publisher, BatchComponentCache batchComponentCache, Settings settings) {
     this.publisher = publisher;
     this.batchComponentCache = batchComponentCache;
     this.settings = settings;
@@ -71,6 +74,11 @@ public class SonarDuplicationsIndex extends AbstractCloneIndex {
     for (Block block : blocks) {
       mem.insert(block);
     }
+    indexedFiles.add(inputFile);
+  }
+
+  public boolean isIndexed(InputFile inputFile) {
+    return indexedFiles.contains(inputFile);
   }
 
   public static boolean isCrossProjectDuplicationEnabled(Settings settings) {

sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java → sonar-batch/src/main/java/org/sonar/batch/phases/AbstractPhaseExecutor.java

@@ -21,27 +21,20 @@ package org.sonar.batch.phases;
 
 import org.sonar.api.batch.SensorContext;
 import org.sonar.api.resources.Project;
-import org.sonar.batch.analysis.DefaultAnalysisMode;
-import org.sonar.batch.cpd.CpdExecutor;
 import org.sonar.batch.events.BatchStepEvent;
 import org.sonar.batch.events.EventBus;
 import org.sonar.batch.index.DefaultIndex;
-import org.sonar.batch.issue.IssueCallback;
 import org.sonar.batch.issue.ignore.scanner.IssueExclusionsLoader;
-import org.sonar.batch.issue.tracking.IssueTransition;
-import org.sonar.batch.report.ReportPublisher;
 import org.sonar.batch.rule.QProfileVerifier;
 import org.sonar.batch.scan.filesystem.DefaultModuleFileSystem;
 import org.sonar.batch.scan.filesystem.FileSystemLogger;
-import org.sonar.batch.scan.report.IssuesReports;
 
-public final class PhaseExecutor {
+public abstract class AbstractPhaseExecutor {
 
   private final EventBus eventBus;
   private final PostJobsExecutor postJobsExecutor;
   private final InitializersExecutor initializersExecutor;
   private final SensorsExecutor sensorsExecutor;
-  private final ReportPublisher reportPublisher;
   private final SensorContext sensorContext;
   private final DefaultIndex index;
   private final ProjectInitializer pi;
@@ -49,41 +42,29 @@ public final class PhaseExecutor {
   private final DefaultModuleFileSystem fs;
   private final QProfileVerifier profileVerifier;
   private final IssueExclusionsLoader issueExclusionsLoader;
-  private final IssuesReports issuesReport;
-  private final DefaultAnalysisMode analysisMode;
-  private final IssueTransition localIssueTracking;
-  private final IssueCallback issueCallback;
-  private final CpdExecutor cpdExecutor;
 
-  public PhaseExecutor(InitializersExecutor initializersExecutor, PostJobsExecutor postJobsExecutor, SensorsExecutor sensorsExecutor,
+  public AbstractPhaseExecutor(InitializersExecutor initializersExecutor, PostJobsExecutor postJobsExecutor, SensorsExecutor sensorsExecutor,
     SensorContext sensorContext, DefaultIndex index,
-    EventBus eventBus, ReportPublisher reportPublisher, ProjectInitializer pi,
-    FileSystemLogger fsLogger, IssuesReports jsonReport, DefaultModuleFileSystem fs, QProfileVerifier profileVerifier,
-    IssueExclusionsLoader issueExclusionsLoader, DefaultAnalysisMode analysisMode, IssueTransition localIssueTracking, IssueCallback issueCallback,
-    CpdExecutor cpdExecutor) {
+    EventBus eventBus, ProjectInitializer pi,
+    FileSystemLogger fsLogger, DefaultModuleFileSystem fs, QProfileVerifier profileVerifier,
+    IssueExclusionsLoader issueExclusionsLoader) {
     this.postJobsExecutor = postJobsExecutor;
     this.initializersExecutor = initializersExecutor;
     this.sensorsExecutor = sensorsExecutor;
     this.sensorContext = sensorContext;
     this.index = index;
     this.eventBus = eventBus;
-    this.reportPublisher = reportPublisher;
     this.pi = pi;
     this.fsLogger = fsLogger;
-    this.issuesReport = jsonReport;
     this.fs = fs;
     this.profileVerifier = profileVerifier;
     this.issueExclusionsLoader = issueExclusionsLoader;
-    this.analysisMode = analysisMode;
-    this.localIssueTracking = localIssueTracking;
-    this.issueCallback = issueCallback;
-    this.cpdExecutor = cpdExecutor;
   }
 
   /**
    * Executed on each module
    */
-  public void execute(Project module) {
+  public final void execute(Project module) {
     pi.execute(module);
 
     eventBus.fireEvent(new ProjectAnalysisEvent(module, true));
@@ -102,54 +83,14 @@ public final class PhaseExecutor {
     sensorsExecutor.execute(sensorContext);
 
     if (module.isRoot()) {
-      if (analysisMode.isIssues()) {
-        localIssueTracking();
-        issuesCallback();
-      } else {
-        computeDuplications();
-      }
-      issuesReport();
-      publishReportJob();
+      executeOnRoot();
       postJobsExecutor.execute(sensorContext);
     }
     cleanMemory();
     eventBus.fireEvent(new ProjectAnalysisEvent(module, false));
   }
 
-  private void computeDuplications() {
-    String stepName = "Computing duplications";
-    eventBus.fireEvent(new BatchStepEvent(stepName, true));
-    cpdExecutor.execute();
-    eventBus.fireEvent(new BatchStepEvent(stepName, false));
-  }
-
-  private void publishReportJob() {
-    String stepName = "Publish report";
-    eventBus.fireEvent(new BatchStepEvent(stepName, true));
-    this.reportPublisher.execute();
-    eventBus.fireEvent(new BatchStepEvent(stepName, false));
-  }
-
-  private void localIssueTracking() {
-    String stepName = "Local Issue Tracking";
-    eventBus.fireEvent(new BatchStepEvent(stepName, true));
-    localIssueTracking.execute();
-    eventBus.fireEvent(new BatchStepEvent(stepName, false));
-  }
-
-  private void issuesCallback() {
-    String stepName = "Issues Callback";
-    eventBus.fireEvent(new BatchStepEvent(stepName, true));
-    issueCallback.execute();
-    eventBus.fireEvent(new BatchStepEvent(stepName, false));
-  }
-
-  private void issuesReport() {
-    String stepName = "Issues Reports";
-    eventBus.fireEvent(new BatchStepEvent(stepName, true));
-    issuesReport.execute();
-    eventBus.fireEvent(new BatchStepEvent(stepName, false));
-  }
+  protected abstract void executeOnRoot();
 
   private void initIssueExclusions() {
     String stepName = "Init issue exclusions";

sonar-batch/src/main/java/org/sonar/batch/phases/IssuesPhaseExecutor.java

@@ -0,0 +1,84 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.phases;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.batch.SensorContext;
+import org.sonar.batch.events.BatchStepEvent;
+import org.sonar.batch.events.EventBus;
+import org.sonar.batch.index.DefaultIndex;
+import org.sonar.batch.issue.IssueCallback;
+import org.sonar.batch.issue.ignore.scanner.IssueExclusionsLoader;
+import org.sonar.batch.issue.tracking.IssueTransition;
+import org.sonar.batch.rule.QProfileVerifier;
+import org.sonar.batch.scan.filesystem.DefaultModuleFileSystem;
+import org.sonar.batch.scan.filesystem.FileSystemLogger;
+import org.sonar.batch.scan.report.IssuesReports;
+
+public final class IssuesPhaseExecutor extends AbstractPhaseExecutor {
+
+  private static final Logger LOG = LoggerFactory.getLogger(IssuesPhaseExecutor.class);
+
+  private final EventBus eventBus;
+  private final IssuesReports issuesReport;
+  private final IssueTransition localIssueTracking;
+  private final IssueCallback issueCallback;
+
+  public IssuesPhaseExecutor(InitializersExecutor initializersExecutor, PostJobsExecutor postJobsExecutor, SensorsExecutor sensorsExecutor, SensorContext sensorContext,
+    DefaultIndex index, EventBus eventBus, ProjectInitializer pi, FileSystemLogger fsLogger, IssuesReports jsonReport, DefaultModuleFileSystem fs, QProfileVerifier profileVerifier,
+    IssueExclusionsLoader issueExclusionsLoader, IssueTransition localIssueTracking, IssueCallback issueCallback) {
+    super(initializersExecutor, postJobsExecutor, sensorsExecutor, sensorContext, index, eventBus, pi, fsLogger, fs, profileVerifier, issueExclusionsLoader);
+    this.eventBus = eventBus;
+    this.issuesReport = jsonReport;
+    this.localIssueTracking = localIssueTracking;
+    this.issueCallback = issueCallback;
+  }
+
+  @Override
+  protected void executeOnRoot() {
+    localIssueTracking();
+    issuesCallback();
+    issuesReport();
+    LOG.info("ANALYSIS SUCCESSFUL");
+  }
+
+  private void localIssueTracking() {
+    String stepName = "Local Issue Tracking";
+    eventBus.fireEvent(new BatchStepEvent(stepName, true));
+    localIssueTracking.execute();
+    eventBus.fireEvent(new BatchStepEvent(stepName, false));
+  }
+
+  private void issuesCallback() {
+    String stepName = "Issues Callback";
+    eventBus.fireEvent(new BatchStepEvent(stepName, true));
+    issueCallback.execute();
+    eventBus.fireEvent(new BatchStepEvent(stepName, false));
+  }
+
+  private void issuesReport() {
+    String stepName = "Issues Reports";
+    eventBus.fireEvent(new BatchStepEvent(stepName, true));
+    issuesReport.execute();
+    eventBus.fireEvent(new BatchStepEvent(stepName, false));
+  }
+
+}

sonar-batch/src/main/java/org/sonar/batch/phases/PublishPhaseExecutor.java

@@ -0,0 +1,67 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.phases;
+
+import org.sonar.api.batch.SensorContext;
+import org.sonar.batch.cpd.CpdExecutor;
+import org.sonar.batch.events.BatchStepEvent;
+import org.sonar.batch.events.EventBus;
+import org.sonar.batch.index.DefaultIndex;
+import org.sonar.batch.issue.ignore.scanner.IssueExclusionsLoader;
+import org.sonar.batch.report.ReportPublisher;
+import org.sonar.batch.rule.QProfileVerifier;
+import org.sonar.batch.scan.filesystem.DefaultModuleFileSystem;
+import org.sonar.batch.scan.filesystem.FileSystemLogger;
+
+public final class PublishPhaseExecutor extends AbstractPhaseExecutor {
+
+  private final EventBus eventBus;
+  private final ReportPublisher reportPublisher;
+  private final CpdExecutor cpdExecutor;
+
+  public PublishPhaseExecutor(InitializersExecutor initializersExecutor, PostJobsExecutor postJobsExecutor, SensorsExecutor sensorsExecutor, SensorContext sensorContext,
+    DefaultIndex index, EventBus eventBus, ReportPublisher reportPublisher, ProjectInitializer pi, FileSystemLogger fsLogger, DefaultModuleFileSystem fs,
+    QProfileVerifier profileVerifier, IssueExclusionsLoader issueExclusionsLoader, CpdExecutor cpdExecutor) {
+    super(initializersExecutor, postJobsExecutor, sensorsExecutor, sensorContext, index, eventBus, pi, fsLogger, fs, profileVerifier, issueExclusionsLoader);
+    this.eventBus = eventBus;
+    this.reportPublisher = reportPublisher;
+    this.cpdExecutor = cpdExecutor;
+  }
+
+  @Override
+  protected void executeOnRoot() {
+    computeDuplications();
+    publishReportJob();
+  }
+
+  private void computeDuplications() {
+    String stepName = "Computing duplications";
+    eventBus.fireEvent(new BatchStepEvent(stepName, true));
+    cpdExecutor.execute();
+    eventBus.fireEvent(new BatchStepEvent(stepName, false));
+  }
+
+  private void publishReportJob() {
+    String stepName = "Publish report";
+    eventBus.fireEvent(new BatchStepEvent(stepName, true));
+    this.reportPublisher.execute();
+    eventBus.fireEvent(new BatchStepEvent(stepName, false));
+  }
+}

sonar-batch/src/main/java/org/sonar/batch/report/MetadataPublisher.java

@@ -23,7 +23,7 @@ import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
 import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Project;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.batch.index.BatchComponent;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.batch.protocol.output.BatchReport;
@@ -50,7 +50,7 @@ public class MetadataPublisher implements ReportPublisherStep {
       .setAnalysisDate(((Project) rootProject.resource()).getAnalysisDate().getTime())
       // Here we want key without branch
       .setProjectKey(root.getKey())
-      .setCrossProjectDuplicationActivated(SonarDuplicationsIndex.isCrossProjectDuplicationEnabled(settings))
+      .setCrossProjectDuplicationActivated(SonarCpdBlockIndex.isCrossProjectDuplicationEnabled(settings))
       .setRootComponentRef(rootProject.batchId());
     String branch = root.properties().get(CoreProperties.PROJECT_BRANCH_PROPERTY);
     if (branch != null) {

sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java

@@ -21,6 +21,7 @@ package org.sonar.batch.scan;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.sonar.api.batch.AnalysisMode;
 import org.sonar.api.batch.InstantiationStrategy;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
 import org.sonar.api.batch.fs.internal.FileMetadata;
@@ -46,10 +47,12 @@ import org.sonar.batch.issue.ignore.pattern.IssueExclusionPatternInitializer;
 import org.sonar.batch.issue.ignore.pattern.IssueInclusionPatternInitializer;
 import org.sonar.batch.issue.ignore.scanner.IssueExclusionsLoader;
 import org.sonar.batch.issue.ignore.scanner.IssueExclusionsRegexpScanner;
+import org.sonar.batch.phases.AbstractPhaseExecutor;
 import org.sonar.batch.phases.InitializersExecutor;
-import org.sonar.batch.phases.PhaseExecutor;
+import org.sonar.batch.phases.IssuesPhaseExecutor;
 import org.sonar.batch.phases.PostJobsExecutor;
 import org.sonar.batch.phases.ProjectInitializer;
+import org.sonar.batch.phases.PublishPhaseExecutor;
 import org.sonar.batch.phases.SensorsExecutor;
 import org.sonar.batch.postjob.DefaultPostJobContext;
 import org.sonar.batch.postjob.PostJobOptimizer;
@@ -104,9 +107,15 @@ public class ModuleScanContainer extends ComponentContainer {
     ModuleSettings moduleSettings = getComponentByType(ModuleSettings.class);
     module.setSettings(moduleSettings);
 
+    if (getComponentByType(AnalysisMode.class).isIssues()) {
+      add(IssuesPhaseExecutor.class,
+        IssuesReports.class);
+    } else {
+      add(PublishPhaseExecutor.class);
+    }
+
     add(
       EventBus.class,
-      PhaseExecutor.class,
       RuleFinderCompatibility.class,
       PostJobsExecutor.class,
       SensorsExecutor.class,
@@ -145,9 +154,6 @@ public class ModuleScanContainer extends ComponentContainer {
       QProfileSensor.class,
       CheckFactory.class,
 
-      // report
-      IssuesReports.class,
-
       // issues
       IssuableFactory.class,
       ModuleIssues.class,
@@ -182,7 +188,7 @@ public class ModuleScanContainer extends ComponentContainer {
     DefaultIndex index = getComponentByType(DefaultIndex.class);
     index.setCurrentProject(module, getComponentByType(DefaultSensorStorage.class));
 
-    getComponentByType(PhaseExecutor.class).execute(module);
+    getComponentByType(AbstractPhaseExecutor.class).execute(module);
 
     // Free memory since module settings are no more used
     module.setSettings(null);

sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java

@@ -42,7 +42,7 @@ import org.sonar.batch.bootstrap.ExtensionUtils;
 import org.sonar.batch.bootstrap.MetricProvider;
 import org.sonar.batch.cache.ProjectPersistentCacheProvider;
 import org.sonar.batch.cpd.CpdExecutor;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.batch.events.EventBus;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.batch.index.Caches;
@@ -209,7 +209,7 @@ public class ProjectScanContainer extends ComponentContainer {
 
       // Cpd
       CpdExecutor.class,
-      SonarDuplicationsIndex.class,
+      SonarCpdBlockIndex.class,
 
       ScanTaskObservers.class,
       UserRepositoryLoader.class);

sonar-batch/src/main/java/org/sonar/batch/scan/report/IssuesReports.java

@@ -19,26 +19,20 @@
  */
 package org.sonar.batch.scan.report;
 
-import org.sonar.batch.analysis.DefaultAnalysisMode;
-
 import org.sonar.api.batch.BatchSide;
 
 @BatchSide
 public class IssuesReports {
 
-  private final DefaultAnalysisMode analysisMode;
   private final Reporter[] reporters;
 
-  public IssuesReports(DefaultAnalysisMode analysisMode, Reporter... reporters) {
+  public IssuesReports(Reporter... reporters) {
     this.reporters = reporters;
-    this.analysisMode = analysisMode;
   }
 
   public void execute() {
-    if (analysisMode.isIssues() || analysisMode.isMediumTest()) {
-      for (Reporter reporter : reporters) {
-        reporter.execute();
-      }
+    for (Reporter reporter : reporters) {
+      reporter.execute();
     }
   }
 }

sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java

@@ -60,8 +60,7 @@ public final class ScmSensor implements Sensor {
 
   @Override
   public void describe(SensorDescriptor descriptor) {
-    descriptor.name("SCM Sensor")
-      .disabledInIssues();
+    descriptor.name("SCM Sensor");
   }
 
   @Override

sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java

@@ -22,14 +22,14 @@ package org.sonar.batch.sensor;
 import java.io.Serializable;
 import org.sonar.api.batch.AnalysisMode;
 import org.sonar.api.batch.fs.FileSystem;
-import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.InputModule;
 import org.sonar.api.batch.rule.ActiveRules;
 import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.batch.sensor.coverage.NewCoverage;
 import org.sonar.api.batch.sensor.coverage.internal.DefaultCoverage;
+import org.sonar.api.batch.sensor.cpd.NewCpdTokens;
+import org.sonar.api.batch.sensor.cpd.internal.DefaultCpdTokens;
 import org.sonar.api.batch.sensor.highlighting.NewHighlighting;
-import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting;
 import org.sonar.api.batch.sensor.internal.SensorStorage;
 import org.sonar.api.batch.sensor.issue.NewIssue;
@@ -37,29 +37,13 @@ import org.sonar.api.batch.sensor.issue.internal.DefaultIssue;
 import org.sonar.api.batch.sensor.measure.NewMeasure;
 import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure;
 import org.sonar.api.config.Settings;
+import org.sonar.batch.sensor.noop.NoOpNewCpdTokens;
+import org.sonar.batch.sensor.noop.NoOpNewHighlighting;
 
 public class DefaultSensorContext implements SensorContext {
 
   private static final NoOpNewHighlighting NO_OP_NEW_HIGHLIGHTING = new NoOpNewHighlighting();
-
-  private static final class NoOpNewHighlighting implements NewHighlighting {
-    @Override
-    public void save() {
-      // Do nothing
-    }
-
-    @Override
-    public NewHighlighting onFile(InputFile inputFile) {
-      // Do nothing
-      return this;
-    }
-
-    @Override
-    public NewHighlighting highlight(int startOffset, int endOffset, TypeOfText typeOfText) {
-      // Do nothing
-      return this;
-    }
-  }
+  private static final NoOpNewCpdTokens NO_OP_NEW_CPD_TOKENS = new NoOpNewCpdTokens();
 
   private final Settings settings;
   private final FileSystem fs;
@@ -92,11 +76,6 @@ public class DefaultSensorContext implements SensorContext {
     return activeRules;
   }
 
-  @Override
-  public AnalysisMode analysisMode() {
-    return analysisMode;
-  }
-
   @Override
   public InputModule module() {
     return module;
@@ -125,4 +104,12 @@ public class DefaultSensorContext implements SensorContext {
     return new DefaultCoverage(sensorStorage);
   }
 
+  @Override
+  public NewCpdTokens newCpdTokens() {
+    if (analysisMode.isIssues()) {
+      return NO_OP_NEW_CPD_TOKENS;
+    }
+    return new DefaultCpdTokens(sensorStorage);
+  }
+
 }

sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.batch.sensor;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import java.util.Arrays;
@@ -37,6 +38,7 @@ import org.sonar.api.batch.measure.MetricFinder;
 import org.sonar.api.batch.rule.ActiveRules;
 import org.sonar.api.batch.sensor.coverage.CoverageType;
 import org.sonar.api.batch.sensor.coverage.internal.DefaultCoverage;
+import org.sonar.api.batch.sensor.cpd.internal.DefaultCpdTokens;
 import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting;
 import org.sonar.api.batch.sensor.highlighting.internal.SyntaxHighlightingRule;
 import org.sonar.api.batch.sensor.internal.SensorStorage;
@@ -51,6 +53,8 @@ import org.sonar.api.resources.Resource;
 import org.sonar.api.source.Symbol;
 import org.sonar.api.utils.KeyValueFormat;
 import org.sonar.api.utils.SonarException;
+import org.sonar.batch.cpd.DefaultCpdBlockIndexer;
+import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.batch.index.BatchComponent;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.batch.issue.ModuleIssues;
@@ -61,6 +65,8 @@ import org.sonar.batch.report.ReportPublisher;
 import org.sonar.batch.scan.measure.MeasureCache;
 import org.sonar.batch.sensor.coverage.CoverageExclusions;
 import org.sonar.batch.source.DefaultSymbol;
+import org.sonar.duplications.block.Block;
+import org.sonar.duplications.internal.pmd.PmdBlockChunker;
 
 public class DefaultSensorStorage implements SensorStorage {
 
@@ -90,16 +96,20 @@ public class DefaultSensorStorage implements SensorStorage {
   private final BatchComponentCache componentCache;
   private final ReportPublisher reportPublisher;
   private final MeasureCache measureCache;
+  private final SonarCpdBlockIndex index;
+  private final Settings settings;
 
   public DefaultSensorStorage(MetricFinder metricFinder, ModuleIssues moduleIssues,
     Settings settings, FileSystem fs, ActiveRules activeRules,
-    CoverageExclusions coverageExclusions, BatchComponentCache componentCache, ReportPublisher reportPublisher, MeasureCache measureCache) {
+    CoverageExclusions coverageExclusions, BatchComponentCache componentCache, ReportPublisher reportPublisher, MeasureCache measureCache, SonarCpdBlockIndex index) {
     this.metricFinder = metricFinder;
     this.moduleIssues = moduleIssues;
+    this.settings = settings;
     this.coverageExclusions = coverageExclusions;
     this.componentCache = componentCache;
     this.reportPublisher = reportPublisher;
     this.measureCache = measureCache;
+    this.index = index;
   }
 
   private Metric findMetricOrFail(String metricKey) {
@@ -254,4 +264,22 @@ public class DefaultSensorStorage implements SensorStorage {
       return builder.build();
     }
   }
+
+  @Override
+  public void store(DefaultCpdTokens defaultCpdTokens) {
+    InputFile inputFile = defaultCpdTokens.inputFile();
+    PmdBlockChunker blockChunker = new PmdBlockChunker(getBlockSize(inputFile.language()));
+    List<Block> blocks = blockChunker.chunk(inputFile.key(), defaultCpdTokens.getTokenLines());
+    index.insert(inputFile, blocks);
+  }
+
+  @VisibleForTesting
+  int getBlockSize(String languageKey) {
+    int blockSize = settings.getInt("sonar.cpd." + languageKey + ".minimumLines");
+    if (blockSize == 0) {
+      blockSize = DefaultCpdBlockIndexer.getDefaultBlockSize(languageKey);
+    }
+    return blockSize;
+  }
+
 }

sonar-batch/src/main/java/org/sonar/batch/sensor/SensorOptimizer.java

@@ -21,7 +21,6 @@ package org.sonar.batch.sensor;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.sonar.api.batch.AnalysisMode;
 import org.sonar.api.batch.BatchSide;
 import org.sonar.api.batch.fs.FilePredicate;
 import org.sonar.api.batch.fs.FileSystem;
@@ -37,13 +36,11 @@ public class SensorOptimizer {
   private final FileSystem fs;
   private final ActiveRules activeRules;
   private final Settings settings;
-  private final AnalysisMode analysisMode;
 
-  public SensorOptimizer(FileSystem fs, ActiveRules activeRules, Settings settings, AnalysisMode analysisMode) {
+  public SensorOptimizer(FileSystem fs, ActiveRules activeRules, Settings settings) {
     this.fs = fs;
     this.activeRules = activeRules;
     this.settings = settings;
-    this.analysisMode = analysisMode;
   }
 
   /**
@@ -62,10 +59,6 @@ public class SensorOptimizer {
       LOG.debug("'{}' skipped because one of the required properties is missing", descriptor.name());
       return false;
     }
-    if (descriptor.isDisabledInIssues() && analysisMode.isIssues()) {
-      LOG.debug("'{}' skipped in issues mode", descriptor.name());
-      return false;
-    }
     return true;
   }
 

sonar-batch/src/main/java/org/sonar/batch/sensor/noop/NoOpNewCpdTokens.java

@@ -0,0 +1,43 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.sensor.noop;
+
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.TextRange;
+import org.sonar.api.batch.sensor.cpd.NewCpdTokens;
+
+public class NoOpNewCpdTokens implements NewCpdTokens {
+  @Override
+  public void save() {
+    // Do nothing
+  }
+
+  @Override
+  public NoOpNewCpdTokens onFile(InputFile inputFile) {
+    // Do nothing
+    return this;
+  }
+
+  @Override
+  public NewCpdTokens addToken(TextRange range, String image) {
+    // Do nothing
+    return this;
+  }
+}

sonar-batch/src/main/java/org/sonar/batch/sensor/noop/NoOpNewHighlighting.java

@@ -0,0 +1,43 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.sensor.noop;
+
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.sensor.highlighting.NewHighlighting;
+import org.sonar.api.batch.sensor.highlighting.TypeOfText;
+
+public class NoOpNewHighlighting implements NewHighlighting {
+  @Override
+  public void save() {
+    // Do nothing
+  }
+
+  @Override
+  public NewHighlighting onFile(InputFile inputFile) {
+    // Do nothing
+    return this;
+  }
+
+  @Override
+  public NewHighlighting highlight(int startOffset, int endOffset, TypeOfText typeOfText) {
+    // Do nothing
+    return this;
+  }
+}

sonar-batch/src/main/java/org/sonar/batch/sensor/noop/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@javax.annotation.ParametersAreNonnullByDefault
+package org.sonar.batch.sensor.noop;

sonar-batch/src/main/java/org/sonar/batch/source/CodeColorizerSensor.java

@@ -44,8 +44,7 @@ public final class CodeColorizerSensor implements Sensor {
 
   @Override
   public void describe(SensorDescriptor descriptor) {
-    descriptor.name("Code Colorizer Sensor")
-      .disabledInIssues();
+    descriptor.name("Code Colorizer Sensor");
   }
 
   @Override

sonar-batch/src/test/java/org/sonar/batch/bootstrap/ExtensionInstallerTest.java

@@ -27,6 +27,7 @@ import org.junit.Test;
 import org.sonar.api.BatchExtension;
 import org.sonar.api.ExtensionProvider;
 import org.sonar.api.SonarPlugin;
+import org.sonar.api.batch.AnalysisMode;
 import org.sonar.core.platform.ComponentContainer;
 import org.sonar.core.platform.PluginInfo;
 
@@ -58,7 +59,7 @@ public class ExtensionInstallerTest {
     when(pluginRepository.getPluginInstance("foo")).thenReturn(newPluginInstance(Foo.class, Bar.class));
 
     ComponentContainer container = new ComponentContainer();
-    ExtensionInstaller installer = new ExtensionInstaller(pluginRepository);
+    ExtensionInstaller installer = new ExtensionInstaller(pluginRepository, mock(AnalysisMode.class));
     installer.install(container, new FooMatcher());
 
     assertThat(container.getComponentByType(Foo.class)).isNotNull();
@@ -70,7 +71,7 @@ public class ExtensionInstallerTest {
     when(pluginRepository.getPluginInfos()).thenReturn(Arrays.asList(new PluginInfo("foo")));
     when(pluginRepository.getPluginInstance("foo")).thenReturn(newPluginInstance(new FooProvider(), new BarProvider()));
     ComponentContainer container = new ComponentContainer();
-    ExtensionInstaller installer = new ExtensionInstaller(pluginRepository);
+    ExtensionInstaller installer = new ExtensionInstaller(pluginRepository, mock(AnalysisMode.class));
 
     installer.install(container, new FooMatcher());
 
@@ -83,7 +84,7 @@ public class ExtensionInstallerTest {
     when(pluginRepository.getPluginInfos()).thenReturn(Arrays.asList(new PluginInfo("foo")));
     when(pluginRepository.getPluginInstance("foo")).thenReturn(newPluginInstance(new FooBarProvider()));
     ComponentContainer container = new ComponentContainer();
-    ExtensionInstaller installer = new ExtensionInstaller(pluginRepository);
+    ExtensionInstaller installer = new ExtensionInstaller(pluginRepository, mock(AnalysisMode.class));
 
     installer.install(container, new TrueMatcher());
 

sonar-batch/src/test/java/org/sonar/batch/cpd/CpdExecutorTest.java

@@ -29,7 +29,7 @@ import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Project;
 import org.sonar.api.utils.log.LogTester;
 import org.sonar.api.utils.log.LoggerLevel;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.batch.index.BatchComponent;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.batch.protocol.output.BatchReportReader;
@@ -55,7 +55,7 @@ import static org.mockito.Mockito.when;
 public class CpdExecutorTest {
   private CpdExecutor executor;
   private Settings settings;
-  private SonarDuplicationsIndex index;
+  private SonarCpdBlockIndex index;
   private ReportPublisher publisher;
   private BatchComponentCache componentCache;
 
@@ -77,7 +77,7 @@ public class CpdExecutorTest {
     File outputDir = temp.newFolder();
 
     settings = new Settings();
-    index = mock(SonarDuplicationsIndex.class);
+    index = mock(SonarCpdBlockIndex.class);
     publisher = mock(ReportPublisher.class);
     when(publisher.getWriter()).thenReturn(new BatchReportWriter(outputDir));
     componentCache = new BatchComponentCache();

sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java

@@ -36,15 +36,15 @@ public class CpdSensorTest {
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
 
-  JavaCpdIndexer sonarEngine;
-  DefaultCpdIndexer sonarBridgeEngine;
+  JavaCpdBlockIndexer sonarEngine;
+  DefaultCpdBlockIndexer sonarBridgeEngine;
   CpdSensor sensor;
   Settings settings;
 
   @Before
   public void setUp() throws IOException {
-    sonarEngine = new JavaCpdIndexer(null, null, null);
-    sonarBridgeEngine = new DefaultCpdIndexer(new CpdMappings(), null, null, null);
+    sonarEngine = new JavaCpdBlockIndexer(null, null, null);
+    sonarBridgeEngine = new DefaultCpdBlockIndexer(new CpdMappings(), null, null, null);
     settings = new Settings(new PropertyDefinitions(CpdComponents.class));
 
     DefaultFileSystem fs = new DefaultFileSystem(temp.newFolder().toPath());

sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdIndexerTest.java → sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdBlockIndexerTest.java

@@ -31,15 +31,15 @@ import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
-public class DefaultCpdIndexerTest {
+public class DefaultCpdBlockIndexerTest {
 
-  private DefaultCpdIndexer engine;
+  private DefaultCpdBlockIndexer engine;
   private Settings settings;
 
   @Before
   public void init() {
     settings = new Settings();
-    engine = new DefaultCpdIndexer(null, null, settings, null);
+    engine = new DefaultCpdBlockIndexer(null, null, settings, null);
   }
 
   @Test
@@ -59,9 +59,9 @@ public class DefaultCpdIndexerTest {
 
   @Test
   public void shouldReturnDefaultBlockSize() {
-    assertThat(DefaultCpdIndexer.getDefaultBlockSize("cobol")).isEqualTo(30);
-    assertThat(DefaultCpdIndexer.getDefaultBlockSize("abap")).isEqualTo(20);
-    assertThat(DefaultCpdIndexer.getDefaultBlockSize("other")).isEqualTo(10);
+    assertThat(DefaultCpdBlockIndexer.getDefaultBlockSize("cobol")).isEqualTo(30);
+    assertThat(DefaultCpdBlockIndexer.getDefaultBlockSize("abap")).isEqualTo(20);
+    assertThat(DefaultCpdBlockIndexer.getDefaultBlockSize("other")).isEqualTo(10);
   }
 
   @Test

sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdIndexerTest.java → sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdBlockIndexerTest.java

@@ -33,7 +33,7 @@ import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.config.Settings;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.duplications.block.Block;
 
@@ -47,17 +47,17 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
 
-public class JavaCpdIndexerTest {
+public class JavaCpdBlockIndexerTest {
   private static final String JAVA = "java";
 
   @Mock
-  private SonarDuplicationsIndex index;
+  private SonarCpdBlockIndex index;
 
   @Captor
   private ArgumentCaptor<List<Block>> blockCaptor;
 
   private Settings settings;
-  private JavaCpdIndexer engine;
+  private JavaCpdBlockIndexer engine;
   private DefaultInputFile file;
 
   @Rule
@@ -77,12 +77,12 @@ public class JavaCpdIndexerTest {
     FileUtils.copyURLToFile(this.getClass().getResource("ManyStatements.java"), ioFile);
 
     settings = new Settings();
-    engine = new JavaCpdIndexer(fs, settings, index);
+    engine = new JavaCpdBlockIndexer(fs, settings, index);
   }
 
   @Test
   public void languageSupported() {
-    JavaCpdIndexer engine = new JavaCpdIndexer(mock(FileSystem.class), new Settings(), index);
+    JavaCpdBlockIndexer engine = new JavaCpdBlockIndexer(mock(FileSystem.class), new Settings(), index);
     assertThat(engine.isLanguageSupported(JAVA)).isTrue();
     assertThat(engine.isLanguageSupported("php")).isFalse();
   }

sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java

@@ -22,6 +22,8 @@ package org.sonar.batch.mediumtest.cpd;
 import com.google.common.collect.ImmutableMap;
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import org.apache.commons.io.FileUtils;
@@ -32,6 +34,9 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.measures.CoreMetrics;
@@ -40,11 +45,20 @@ import org.sonar.batch.mediumtest.TaskResult;
 import org.sonar.batch.protocol.output.BatchReport;
 import org.sonar.batch.protocol.output.BatchReport.Measure;
 import org.sonar.xoo.XooPlugin;
+import org.sonar.xoo.lang.CpdTokenizerSensor;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+@RunWith(Parameterized.class)
 public class CpdMediumTest {
 
+  @Parameters(name = "new api: {0}")
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[][] {
+      {true}, {false}
+    });
+  }
+
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
 
@@ -60,6 +74,12 @@ public class CpdMediumTest {
 
   private ImmutableMap.Builder<String, String> builder;
 
+  private boolean useNewSensorApi;
+
+  public CpdMediumTest(boolean useNewSensorApi) {
+    this.useNewSensorApi = useNewSensorApi;
+  }
+
   @Before
   public void prepare() throws IOException {
     tester.start();
@@ -73,6 +93,9 @@ public class CpdMediumTest {
       .put("sonar.projectName", "Foo Project")
       .put("sonar.projectVersion", "1.0-SNAPSHOT")
       .put("sonar.projectDescription", "Description of Foo Project");
+    if (useNewSensorApi) {
+      builder.put(CpdTokenizerSensor.ENABLE_PROP, "true");
+    }
   }
 
   @After

sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorContextTest.java

@@ -71,7 +71,6 @@ public class DefaultSensorContextTest {
     assertThat(adaptor.activeRules()).isEqualTo(activeRules);
     assertThat(adaptor.fileSystem()).isEqualTo(fs);
     assertThat(adaptor.settings()).isEqualTo(settings);
-    assertThat(adaptor.analysisMode()).isEqualTo(analysisMode);
 
     assertThat(adaptor.newIssue()).isNotNull();
     assertThat(adaptor.newMeasure()).isNotNull();

sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java

@@ -39,6 +39,7 @@ import org.sonar.api.measures.Measure;
 import org.sonar.api.resources.File;
 import org.sonar.api.resources.Project;
 import org.sonar.api.resources.Resource;
+import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.batch.issue.ModuleIssues;
 import org.sonar.batch.report.ReportPublisher;
@@ -84,7 +85,7 @@ public class DefaultSensorStorageTest {
     when(coverageExclusions.accept(any(Resource.class), any(Measure.class))).thenReturn(true);
     resourceCache = new BatchComponentCache();
     sensorStorage = new DefaultSensorStorage(metricFinder,
-      moduleIssues, settings, fs, activeRules, coverageExclusions, resourceCache, mock(ReportPublisher.class), measureCache);
+      moduleIssues, settings, fs, activeRules, coverageExclusions, resourceCache, mock(ReportPublisher.class), measureCache, mock(SonarCpdBlockIndex.class));
   }
 
   @Test

sonar-batch/src/test/java/org/sonar/batch/sensor/SensorOptimizerTest.java

@@ -24,7 +24,6 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.api.batch.AnalysisMode;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
@@ -35,8 +34,6 @@ import org.sonar.api.config.Settings;
 import org.sonar.api.rule.RuleKey;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class SensorOptimizerTest {
 
@@ -49,14 +46,12 @@ public class SensorOptimizerTest {
   private DefaultFileSystem fs;
   private SensorOptimizer optimizer;
   private Settings settings;
-  private AnalysisMode analysisMode;
 
   @Before
   public void prepare() throws Exception {
     fs = new DefaultFileSystem(temp.newFolder().toPath());
     settings = new Settings();
-    analysisMode = mock(AnalysisMode.class);
-    optimizer = new SensorOptimizer(fs, new ActiveRulesBuilder().build(), settings, analysisMode);
+    optimizer = new SensorOptimizer(fs, new ActiveRulesBuilder().build(), settings);
   }
 
   @Test
@@ -114,7 +109,7 @@ public class SensorOptimizerTest {
       .create(RuleKey.of("repo1", "foo"))
       .activate()
       .build();
-    optimizer = new SensorOptimizer(fs, activeRules, settings, analysisMode);
+    optimizer = new SensorOptimizer(fs, activeRules, settings);
 
     assertThat(optimizer.shouldExecute(descriptor)).isFalse();
 
@@ -124,7 +119,7 @@ public class SensorOptimizerTest {
       .create(RuleKey.of("squid", "rule"))
       .activate()
       .build();
-    optimizer = new SensorOptimizer(fs, activeRules, settings, analysisMode);
+    optimizer = new SensorOptimizer(fs, activeRules, settings);
     assertThat(optimizer.shouldExecute(descriptor)).isTrue();
   }
 
@@ -138,15 +133,4 @@ public class SensorOptimizerTest {
     assertThat(optimizer.shouldExecute(descriptor)).isTrue();
   }
 
-  @Test
-  public void should_disabled_in_issues_mode() {
-    DefaultSensorDescriptor descriptor = new DefaultSensorDescriptor()
-      .disabledInIssues();
-    assertThat(optimizer.shouldExecute(descriptor)).isTrue();
-
-    when(analysisMode.isIssues()).thenReturn(true);
-
-    assertThat(optimizer.shouldExecute(descriptor)).isFalse();
-  }
-
 }

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorContext.java

@@ -21,11 +21,11 @@ package org.sonar.api.batch.sensor;
 
 import com.google.common.annotations.Beta;
 import java.io.Serializable;
-import org.sonar.api.batch.AnalysisMode;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputModule;
 import org.sonar.api.batch.rule.ActiveRules;
 import org.sonar.api.batch.sensor.coverage.NewCoverage;
+import org.sonar.api.batch.sensor.cpd.NewCpdTokens;
 import org.sonar.api.batch.sensor.highlighting.NewHighlighting;
 import org.sonar.api.batch.sensor.internal.SensorContextTester;
 import org.sonar.api.batch.sensor.issue.Issue;
@@ -57,11 +57,6 @@ public interface SensorContext {
    */
   ActiveRules activeRules();
 
-  /**
-   * Get analysis mode.
-   */
-  AnalysisMode analysisMode();
-
   /**
    * @since 5.5
    */
@@ -100,4 +95,13 @@ public interface SensorContext {
    */
   NewCoverage newCoverage();
 
+  // ------------ CPD ------------
+
+  /**
+   * Builder to define CPD tokens in a file.
+   * Don't forget to call {@link NewCpdTokens#save()}.
+   * @since 5.5
+   */
+  NewCpdTokens newCpdTokens();
+
 }

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorDescriptor.java

@@ -81,9 +81,4 @@ public interface SensorDescriptor {
    */
   SensorDescriptor requireProperties(String... propertyKeys);
 
-  /**
-   * Should this sensor be disabled in issues mode. Default is to run all sensors in issues mode.
-   */
-  SensorDescriptor disabledInIssues();
-
 }

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cpd/NewCpdTokens.java

@@ -0,0 +1,49 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.sensor.cpd;
+
+import com.google.common.annotations.Beta;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.TextRange;
+
+/**
+ * This builder is used to define tokens used by CPD algorithm on files.
+ * @since 5.5
+ */
+@Beta
+public interface NewCpdTokens {
+
+  /**
+   * The tokenized file.
+   */
+  NewCpdTokens onFile(InputFile inputFile);
+
+  /**
+   * Call this method to register a token in a range. Tokens should be registered in order.
+   * @param range Token position. Use {@link InputFile#newRange(int, int, int, int)} to get a valid range.
+   * @param image Text content of the token. Can be replaced by a constant placeholder for some tokens (like litterals).
+   */
+  NewCpdTokens addToken(TextRange range, String image);
+
+  /**
+   * Call this method only once when your are done with defining tokens of the file.
+   */
+  void save();
+}

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cpd/internal/DefaultCpdTokens.java

@@ -0,0 +1,96 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.sensor.cpd.internal;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.TextRange;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.sensor.cpd.NewCpdTokens;
+import org.sonar.api.batch.sensor.internal.DefaultStorable;
+import org.sonar.api.batch.sensor.internal.SensorStorage;
+import org.sonar.duplications.internal.pmd.TokensLine;
+
+public class DefaultCpdTokens extends DefaultStorable implements NewCpdTokens {
+
+  private final ImmutableList.Builder<TokensLine> result = ImmutableList.builder();
+  private DefaultInputFile inputFile;
+  private int startLine = Integer.MIN_VALUE;
+  private int startIndex = 0;
+  private int currentIndex = 0;
+  private StringBuilder sb = new StringBuilder();
+  private TextRange lastRange;
+
+  public DefaultCpdTokens(SensorStorage storage) {
+    super(storage);
+  }
+
+  @Override
+  public DefaultCpdTokens onFile(InputFile inputFile) {
+    Preconditions.checkNotNull(inputFile, "file can't be null");
+    this.inputFile = (DefaultInputFile) inputFile;
+    return this;
+  }
+
+  public InputFile inputFile() {
+    return inputFile;
+  }
+
+  @Override
+  public NewCpdTokens addToken(TextRange range, String image) {
+    Preconditions.checkNotNull(range, "Range should not be null");
+    Preconditions.checkState(inputFile != null, "Call onFile() first");
+    Preconditions.checkState(lastRange == null || lastRange.end().compareTo(range.start()) >= 0,
+      "Tokens of file %s should be provided in order. \nPrevious token: %s\nLast token: %s", inputFile, lastRange, range);
+
+    String value = image;
+
+    int line = range.start().line();
+    if (line != startLine) {
+      addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
+      startIndex = currentIndex + 1;
+      startLine = line;
+    }
+    currentIndex++;
+    sb.append(value);
+
+    return this;
+  }
+
+  public List<TokensLine> getTokenLines() {
+    return result.build();
+  }
+
+  private static void addNewTokensLine(ImmutableList.Builder<TokensLine> result, int startUnit, int endUnit, int startLine, StringBuilder sb) {
+    if (sb.length() != 0) {
+      result.add(new TokensLine(startUnit, endUnit, startLine, sb.toString()));
+      sb.setLength(0);
+    }
+  }
+
+  @Override
+  protected void doSave() {
+    Preconditions.checkState(inputFile != null, "Call onFile() first");
+    addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
+    storage.store(this);
+  }
+}

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cpd/internal/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@javax.annotation.ParametersAreNonnullByDefault
+package org.sonar.api.batch.sensor.cpd.internal;

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cpd/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@javax.annotation.ParametersAreNonnullByDefault
+package org.sonar.api.batch.sensor.cpd;

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/DefaultSensorDescriptor.java

@@ -32,7 +32,6 @@ public class DefaultSensorDescriptor implements SensorDescriptor {
   private InputFile.Type type = null;
   private String[] ruleRepositories = new String[0];
   private String[] properties = new String[0];
-  private boolean disabledInIssues = false;
 
   public String name() {
     return name;
@@ -55,10 +54,6 @@ public class DefaultSensorDescriptor implements SensorDescriptor {
     return Arrays.asList(properties);
   }
 
-  public boolean isDisabledInIssues() {
-    return disabledInIssues;
-  }
-
   @Override
   public DefaultSensorDescriptor name(String name) {
     this.name = name;
@@ -104,10 +99,4 @@ public class DefaultSensorDescriptor implements SensorDescriptor {
     return this;
   }
 
-  @Override
-  public DefaultSensorDescriptor disabledInIssues() {
-    this.disabledInIssues = true;
-    return this;
-  }
-
 }

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java

@@ -44,6 +44,8 @@ import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.batch.sensor.coverage.CoverageType;
 import org.sonar.api.batch.sensor.coverage.NewCoverage;
 import org.sonar.api.batch.sensor.coverage.internal.DefaultCoverage;
+import org.sonar.api.batch.sensor.cpd.NewCpdTokens;
+import org.sonar.api.batch.sensor.cpd.internal.DefaultCpdTokens;
 import org.sonar.api.batch.sensor.highlighting.NewHighlighting;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting;
@@ -71,7 +73,6 @@ public class SensorContextTester implements SensorContext {
   private Settings settings;
   private DefaultFileSystem fs;
   private ActiveRules activeRules;
-  private MockAnalysisMode analysisMode;
   private InMemorySensorStorage sensorStorage;
   private InputModule module;
 
@@ -79,7 +80,6 @@ public class SensorContextTester implements SensorContext {
     this.settings = new Settings();
     this.fs = new DefaultFileSystem(moduleBaseDir);
     this.activeRules = new ActiveRulesBuilder().build();
-    this.analysisMode = new MockAnalysisMode();
     this.sensorStorage = new InMemorySensorStorage();
     this.module = new DefaultInputModule("projectKey");
   }
@@ -115,11 +115,6 @@ public class SensorContextTester implements SensorContext {
     this.activeRules = activeRules;
   }
 
-  @Override
-  public MockAnalysisMode analysisMode() {
-    return analysisMode;
-  }
-
   @Override
   public InputModule module() {
     return module;
@@ -197,6 +192,11 @@ public class SensorContextTester implements SensorContext {
     return new DefaultCoverage(sensorStorage);
   }
 
+  @Override
+  public NewCpdTokens newCpdTokens() {
+    return new DefaultCpdTokens(sensorStorage);
+  }
+
   public List<TypeOfText> highlightingTypeAt(String componentKey, int line, int lineOffset) {
     DefaultHighlighting syntaxHighlightingData = sensorStorage.highlightingByComponent.get(componentKey);
     if (syntaxHighlightingData == null) {
@@ -247,6 +247,7 @@ public class SensorContextTester implements SensorContext {
     private Collection<Issue> allIssues = new ArrayList<>();
 
     private Map<String, DefaultHighlighting> highlightingByComponent = new HashMap<>();
+    private Map<String, DefaultCpdTokens> cpdTokensByComponent = new HashMap<>();
     private Map<String, Map<CoverageType, DefaultCoverage>> coverageByComponent = new HashMap<>();
 
     @Override
@@ -273,6 +274,11 @@ public class SensorContextTester implements SensorContext {
       coverageByComponent.get(key).put(defaultCoverage.type(), defaultCoverage);
     }
 
+    @Override
+    public void store(DefaultCpdTokens defaultCpdTokens) {
+      cpdTokensByComponent.put(defaultCpdTokens.inputFile().key(), defaultCpdTokens);
+    }
+
   }
 
 }

sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorStorage.java

@@ -21,6 +21,7 @@ package org.sonar.api.batch.sensor.internal;
 
 import org.sonar.api.batch.BatchSide;
 import org.sonar.api.batch.sensor.coverage.internal.DefaultCoverage;
+import org.sonar.api.batch.sensor.cpd.internal.DefaultCpdTokens;
 import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting;
 import org.sonar.api.batch.sensor.issue.Issue;
 import org.sonar.api.batch.sensor.measure.Measure;
@@ -43,4 +44,9 @@ public interface SensorStorage {
    */
   void store(DefaultCoverage defaultCoverage);
 
+  /**
+   * @since 5.5 
+   */
+  void store(DefaultCpdTokens defaultCpdTokens);
+
 }

sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/internal/DefaultSensorDescriptorTest.java

@@ -43,13 +43,4 @@ public class DefaultSensorDescriptorTest {
     assertThat(descriptor.ruleRepositories()).containsOnly("squid-java");
   }
 
-  @Test
-  public void disabledAnalysisModes() {
-    DefaultSensorDescriptor descriptor = new DefaultSensorDescriptor();
-    descriptor
-      .disabledInIssues();
-
-    assertThat(descriptor.isDisabledInIssues()).isTrue();
-  }
-
 }

sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java

@@ -81,13 +81,6 @@ public class SensorContextTesterTest {
     assertThat(tester.fileSystem().baseDir()).isNotEqualTo(baseDir);
   }
 
-  @Test
-  public void testAnalysisMode() {
-    assertThat(tester.analysisMode().isPreview()).isFalse();
-    tester.analysisMode().setPreview(true);
-    assertThat(tester.analysisMode().isPreview()).isTrue();
-  }
-
   @Test
   public void testIssues() {
     assertThat(tester.allIssues()).isEmpty();