SONAR-7389 Add new Sensor API to provide duplication tokens

3 files changed, 97 insertions, 3 deletions

diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
index 3da7721c448..679d3c10897 100644
--- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
@@ -27,6 +27,7 @@ import org.sonar.xoo.coverage.OverallCoverageSensor;
 import org.sonar.xoo.coverage.UtCoverageSensor;
 import org.sonar.xoo.extensions.XooPostJob;
 import org.sonar.xoo.extensions.XooProjectBuilder;
+import org.sonar.xoo.lang.CpdTokenizerSensor;
 import org.sonar.xoo.lang.MeasureSensor;
 import org.sonar.xoo.lang.SymbolReferencesSensor;
 import org.sonar.xoo.lang.SyntaxHighlightingSensor;
@@ -97,6 +98,7 @@ public class XooPlugin extends SonarPlugin {
       ChecksSensor.class,
       RandomAccessSensor.class,
       DeprecatedResourceApiSensor.class,
+      CpdTokenizerSensor.class,
 
       OneBlockerIssuePerFileSensor.class,
       OneIssuePerLineSensor.class,
diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/CpdTokenizerSensor.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/CpdTokenizerSensor.java
new file mode 100644
index 00000000000..aa2c69faa57
--- /dev/null
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/CpdTokenizerSensor.java
@@ -0,0 +1,88 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.xoo.lang;
+
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import org.sonar.api.batch.fs.FilePredicates;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.InputFile.Type;
+import org.sonar.api.batch.sensor.Sensor;
+import org.sonar.api.batch.sensor.SensorContext;
+import org.sonar.api.batch.sensor.SensorDescriptor;
+import org.sonar.api.batch.sensor.cpd.NewCpdTokens;
+import org.sonar.xoo.Xoo;
+
+/**
+ * Tokenize files for CPD
+ */
+public class CpdTokenizerSensor implements Sensor {
+
+  public static final String ENABLE_PROP = "sonar.xoo.useNewCpdTokenizerApi";
+
+  private void tokenize(InputFile inputFile, SensorContext context) {
+    int lineIdx = 1;
+    NewCpdTokens newCpdTokens = context.newCpdTokens().onFile(inputFile);
+    try {
+      StringBuilder sb = new StringBuilder();
+      for (String line : FileUtils.readLines(inputFile.file(), context.fileSystem().encoding())) {
+        int startOffset = 0;
+        int endOffset = 0;
+        for (int i = 0; i < line.length(); i++) {
+          char c = line.charAt(i);
+          if (Character.isWhitespace(c)) {
+            if (sb.length() > 0) {
+              newCpdTokens.addToken(inputFile.newRange(lineIdx, startOffset, lineIdx, endOffset), sb.toString());
+              sb.setLength(0);
+            }
+            startOffset = endOffset;
+          } else {
+            sb.append(c);
+          }
+          endOffset++;
+        }
+        if (sb.length() > 0) {
+          newCpdTokens.addToken(inputFile.newRange(lineIdx, startOffset, lineIdx, endOffset), sb.toString());
+          sb.setLength(0);
+        }
+        lineIdx++;
+      }
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to tokenize", e);
+    }
+    newCpdTokens.save();
+  }
+
+  @Override
+  public void describe(SensorDescriptor descriptor) {
+    descriptor
+      .name("Xoo Cpd Tokenizer Sensor")
+      .requireProperty(ENABLE_PROP)
+      .onlyOnLanguages(Xoo.KEY);
+  }
+
+  @Override
+  public void execute(SensorContext context) {
+    FilePredicates p = context.fileSystem().predicates();
+    for (InputFile file : context.fileSystem().inputFiles(p.and(p.hasLanguages(Xoo.KEY), p.hasType(Type.MAIN)))) {
+      tokenize(file, context);
+    }
+  }
+}
diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java
index 74babada4e0..7773d36df39 100644
--- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java
@@ -20,6 +20,8 @@
 package org.sonar.xoo.lang;
 
 import com.google.common.base.Splitter;
+import java.io.File;
+import java.io.IOException;
 import net.sourceforge.pmd.cpd.SourceCode;
 import net.sourceforge.pmd.cpd.TokenEntry;
 import net.sourceforge.pmd.cpd.Tokenizer;
@@ -27,13 +29,14 @@ import net.sourceforge.pmd.cpd.Tokens;
 import org.apache.commons.io.FileUtils;
 import org.sonar.api.batch.BatchSide;
 import org.sonar.api.batch.fs.FileSystem;
-
-import java.io.File;
-import java.io.IOException;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 
 @BatchSide
 public class XooTokenizer implements Tokenizer {
 
+  private static final Logger LOG = Loggers.get(XooTokenizer.class);
+
   private FileSystem fs;
 
   public XooTokenizer(FileSystem fs) {
@@ -43,6 +46,7 @@ public class XooTokenizer implements Tokenizer {
   @Override
   public final void tokenize(SourceCode source, Tokens cpdTokens) {
     String fileName = source.getFileName();
+    LOG.info("Using deprecated tokenizer extension point to tokenize {}", fileName);
     int lineIdx = 1;
     try {
       for (String line : FileUtils.readLines(new File(fileName), fs.encoding())) {