SONAR-6000 Try to decrease size of duplications in persistit

6 files changed, 124 insertions, 119 deletions

diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
index db5d607938a..50fe968725c 100644
--- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
@@ -26,7 +26,8 @@ import org.sonar.xoo.lang.MeasureSensor;
 import org.sonar.xoo.lang.SymbolReferencesSensor;
 import org.sonar.xoo.lang.SyntaxHighlightingSensor;
 import org.sonar.xoo.lang.TestCaseSensor;
-import org.sonar.xoo.lang.XooTokenizerSensor;
+import org.sonar.xoo.lang.XooCpdMapping;
+import org.sonar.xoo.lang.XooTokenizer;
 import org.sonar.xoo.rule.ChecksSensor;
 import org.sonar.xoo.rule.CreateIssueByInternalKeySensor;
 import org.sonar.xoo.rule.DeprecatedResourceApiSensor;
@@ -67,11 +68,14 @@ public class XooPlugin extends SonarPlugin {
       XooScmProvider.class,
       XooBlameCommand.class,
 
+      // CPD
+      XooCpdMapping.class,
+      XooTokenizer.class,
+
       // sensors
       MeasureSensor.class,
       SyntaxHighlightingSensor.class,
       SymbolReferencesSensor.class,
-      XooTokenizerSensor.class,
       TestCaseSensor.class,
       CoveragePerTestSensor.class,
       DependencySensor.class,
diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooCpdMapping.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooCpdMapping.java
new file mode 100644
index 00000000000..a97cf4b8a4b
--- /dev/null
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooCpdMapping.java
@@ -0,0 +1,46 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.xoo.lang;
+
+import net.sourceforge.pmd.cpd.Tokenizer;
+import org.sonar.api.batch.AbstractCpdMapping;
+import org.sonar.api.resources.Language;
+import org.sonar.xoo.Xoo;
+
+public class XooCpdMapping extends AbstractCpdMapping {
+
+  private Xoo xoo;
+  private XooTokenizer xooTokenizer;
+
+  public XooCpdMapping(Xoo xoo, XooTokenizer xooTokenizer) {
+    this.xoo = xoo;
+    this.xooTokenizer = xooTokenizer;
+  }
+
+  @Override
+  public Tokenizer getTokenizer() {
+    return xooTokenizer;
+  }
+
+  @Override
+  public Language getLanguage() {
+    return xoo;
+  }
+}
diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java
new file mode 100644
index 00000000000..9316a4c0f89
--- /dev/null
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java
@@ -0,0 +1,58 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.xoo.lang;
+
+import com.google.common.base.Splitter;
+import net.sourceforge.pmd.cpd.SourceCode;
+import net.sourceforge.pmd.cpd.TokenEntry;
+import net.sourceforge.pmd.cpd.Tokenizer;
+import net.sourceforge.pmd.cpd.Tokens;
+import org.apache.commons.io.FileUtils;
+import org.sonar.api.BatchComponent;
+import org.sonar.api.batch.fs.FileSystem;
+
+import java.io.File;
+import java.io.IOException;
+
+public class XooTokenizer implements Tokenizer, BatchComponent {
+
+  private FileSystem fs;
+
+  public XooTokenizer(FileSystem fs) {
+    this.fs = fs;
+  }
+
+  public final void tokenize(SourceCode source, Tokens cpdTokens) {
+    String fileName = source.getFileName();
+    int lineIdx = 1;
+    try {
+      for (String line : FileUtils.readLines(new File(fileName), fs.encoding())) {
+        for (String token : Splitter.on(" ").split(line)) {
+          TokenEntry cpdToken = new TokenEntry(token, fileName, lineIdx);
+          cpdTokens.add(cpdToken);
+        }
+        lineIdx++;
+      }
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to tokenize", e);
+    }
+    cpdTokens.add(TokenEntry.getEOF());
+  }
+}
diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizerSensor.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizerSensor.java
deleted file mode 100644
index 3190c9d7712..00000000000
--- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizerSensor.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.xoo.lang;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.FilePredicates;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.sensor.Sensor;
-import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.batch.sensor.SensorDescriptor;
-import org.sonar.api.batch.sensor.duplication.DuplicationTokenBuilder;
-import org.sonar.xoo.Xoo;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Tokenize xoo files (separator is whitespace) for duplication detection
- */
-public class XooTokenizerSensor implements Sensor {
-
-  private void computeTokens(InputFile inputFile, SensorContext context) {
-    DuplicationTokenBuilder tokenBuilder = context.duplicationTokenBuilder(inputFile);
-    File ioFile = inputFile.file();
-    int lineId = 0;
-    try {
-      for (String line : FileUtils.readLines(ioFile)) {
-        lineId++;
-        for (String token : Splitter.on(" ").split(line)) {
-          tokenBuilder.addToken(lineId, token);
-        }
-      }
-      tokenBuilder.done();
-    } catch (IOException e) {
-      throw new IllegalStateException("Unable to read file " + ioFile, e);
-    }
-  }
-
-  @Override
-  public void describe(SensorDescriptor descriptor) {
-    descriptor
-      .name("Xoo Tokenizer Sensor")
-      .onlyOnLanguages(Xoo.KEY)
-      .onlyOnFileType(InputFile.Type.MAIN);
-  }
-
-  @Override
-  public void execute(SensorContext context) {
-    String[] cpdExclusions = context.settings().getStringArray(CoreProperties.CPD_EXCLUSIONS);
-    FilePredicates p = context.fileSystem().predicates();
-    List<InputFile> sourceFiles = Lists.newArrayList(context.fileSystem().inputFiles(p.and(
-      p.hasType(InputFile.Type.MAIN),
-      p.hasLanguage(Xoo.KEY),
-      p.doesNotMatchPathPatterns(cpdExclusions)
-      )));
-    if (sourceFiles.isEmpty()) {
-      return;
-    }
-    for (InputFile file : sourceFiles) {
-      computeTokens(file, context);
-    }
-  }
-}
diff --git a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/XooPluginTest.java b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/XooPluginTest.java
index efa35f294d5..9fdbfddfb76 100644
--- a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/XooPluginTest.java
+++ b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/XooPluginTest.java
@@ -27,6 +27,6 @@ public class XooPluginTest {
 
   @Test
   public void provide_extensions() {
-    assertThat(new XooPlugin().getExtensions()).hasSize(21);
+    assertThat(new XooPlugin().getExtensions().size()).isGreaterThan(0);
   }
 }
diff --git a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerSensorTest.java b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerTest.java
index 1fe44615f9b..c3ebec2c6bc 100644
--- a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerSensorTest.java
+++ b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerTest.java
@@ -19,32 +19,28 @@
  */
 package org.sonar.xoo.lang;
 
+import net.sourceforge.pmd.cpd.SourceCode;
+import net.sourceforge.pmd.cpd.TokenEntry;
+import net.sourceforge.pmd.cpd.Tokens;
 import org.apache.commons.io.FileUtils;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.batch.sensor.duplication.DuplicationTokenBuilder;
-import org.sonar.api.batch.sensor.internal.DefaultSensorDescriptor;
 import org.sonar.api.config.Settings;
 
 import java.io.File;
 import java.io.IOException;
 
-import static org.mockito.Matchers.any;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-public class XooTokenizerSensorTest {
+public class XooTokenizerTest {
 
-  private XooTokenizerSensor sensor;
   private SensorContext context = mock(SensorContext.class);
   private DefaultFileSystem fileSystem;
 
@@ -56,7 +52,6 @@ public class XooTokenizerSensorTest {
   @Before
   public void prepare() throws IOException {
     baseDir = temp.newFolder();
-    sensor = new XooTokenizerSensor();
     fileSystem = new DefaultFileSystem(baseDir.toPath());
     when(context.fileSystem()).thenReturn(fileSystem);
     settings = new Settings();
@@ -64,34 +59,20 @@ public class XooTokenizerSensorTest {
   }
 
   @Test
-  public void testDescriptor() {
-    sensor.describe(new DefaultSensorDescriptor());
-  }
-
-  @Test
-  public void testNoExecutionIfExclusion() {
-    DefaultInputFile inputFile = new DefaultInputFile("foo", "src/foo.xoo").setLanguage("xoo");
-    fileSystem.add(inputFile);
-    settings.setProperty(CoreProperties.CPD_EXCLUSIONS, "**/foo.xoo");
-    sensor.execute(context);
-    verify(context, never()).duplicationTokenBuilder(any(InputFile.class));
-  }
-
-  @Test
   public void testExecution() throws IOException {
     File source = new File(baseDir, "src/foo.xoo");
     FileUtils.write(source, "token1 token2 token3\ntoken4");
     DefaultInputFile inputFile = new DefaultInputFile("foo", "src/foo.xoo").setLanguage("xoo");
     fileSystem.add(inputFile);
-    DuplicationTokenBuilder builder = mock(DuplicationTokenBuilder.class);
-    when(context.duplicationTokenBuilder(inputFile)).thenReturn(builder);
 
-    sensor.execute(context);
+    XooTokenizer tokenizer = new XooTokenizer(fileSystem);
+    SourceCode sourceCode = mock(SourceCode.class);
+    when(sourceCode.getFileName()).thenReturn(inputFile.absolutePath());
+    Tokens cpdTokens = new Tokens();
+    tokenizer.tokenize(sourceCode, cpdTokens);
 
-    verify(builder).addToken(1, "token1");
-    verify(builder).addToken(1, "token2");
-    verify(builder).addToken(1, "token3");
-    verify(builder).addToken(2, "token4");
-    verify(builder).done();
+    // 4 tokens + EOF
+    assertThat(cpdTokens.getTokens()).hasSize(5);
+    assertThat(cpdTokens.getTokens().get(3)).isEqualTo(new TokenEntry("token4", "src/foo.xoo", 2));
   }
 }