SONAR-3181,SONAR-3139 Enable cross project CPD for all languages

sonar.cpd.minimumTokens can't be used, because for index in database
tokens must be grouped by lines.

7 files changed, 107 insertions, 152 deletions

diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/CpdPlugin.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/CpdPlugin.java
index b3cb51fa20d..029b5607b49 100644
--- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/CpdPlugin.java
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/CpdPlugin.java
@@ -25,6 +25,7 @@ import org.sonar.api.Property;
 import org.sonar.api.SonarPlugin;
 import org.sonar.plugins.cpd.decorators.DuplicationDensityDecorator;
 import org.sonar.plugins.cpd.decorators.SumDuplicationsDecorator;
+import org.sonar.plugins.cpd.index.IndexFactory;
 
 import java.util.Arrays;
 import java.util.List;
@@ -89,6 +90,7 @@ public class CpdPlugin extends SonarPlugin {
 
   public List getExtensions() {
     return Arrays.asList(CpdSensor.class, SumDuplicationsDecorator.class, DuplicationDensityDecorator.class,
+        IndexFactory.class,
         SonarEngine.class,
         PmdEngine.class,
         SonarBridgeEngine.class);
diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarBridgeEngine.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarBridgeEngine.java
index dc6ee98e725..a1cab7980d3 100644
--- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarBridgeEngine.java
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarBridgeEngine.java
@@ -19,31 +19,33 @@
  */
 package org.sonar.plugins.cpd;
 
-import org.apache.commons.configuration.Configuration;
-import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.CpdMapping;
 import org.sonar.api.batch.SensorContext;
-import org.sonar.api.database.model.ResourceModel;
 import org.sonar.api.resources.*;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.block.BlockChunker;
 import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm;
 import org.sonar.duplications.index.CloneGroup;
-import org.sonar.duplications.index.CloneIndex;
-import org.sonar.duplications.index.PackedMemoryCloneIndex;
+import org.sonar.duplications.internal.pmd.TokenizerBridge;
+import org.sonar.plugins.cpd.index.IndexFactory;
+import org.sonar.plugins.cpd.index.SonarDuplicationsIndex;
 
 import java.util.Collection;
 import java.util.List;
 
 public class SonarBridgeEngine extends CpdEngine {
 
+  private static final int BLOCK_SIZE = 10;
+
+  private final IndexFactory indexFactory;
   private final CpdMapping[] mappings;
 
-  public SonarBridgeEngine() {
-    this.mappings = null;
+  public SonarBridgeEngine(IndexFactory indexFactory) {
+    this(indexFactory, null);
   }
 
-  public SonarBridgeEngine(CpdMapping[] mappings) {
+  public SonarBridgeEngine(IndexFactory indexFactory, CpdMapping[] mappings) {
+    this.indexFactory = indexFactory;
     this.mappings = mappings;
   }
 
@@ -63,43 +65,29 @@ public class SonarBridgeEngine extends CpdEngine {
     CpdMapping mapping = getMapping(project.getLanguage());
 
     // Create index
-    BlockChunker blockChunker = new BlockChunker(getMinimumTokens(project));
-    CloneIndex index = new PackedMemoryCloneIndex();
+    SonarDuplicationsIndex index = indexFactory.create(project);
+
+    BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE);
     TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), fileSystem.getSourceCharset().name());
     for (InputFile inputFile : inputFiles) {
       Resource resource = mapping.createResource(inputFile.getFile(), fileSystem.getSourceDirs());
-      String resourceId = getFullKey(project, resource);
+      String resourceId = SonarEngine.getFullKey(project, resource);
       List<Block> blocks = blockChunker.chunk(resourceId, bridge.tokenize(inputFile.getFile()));
-      for (Block block : blocks) {
-        index.insert(block);
-      }
+      index.insert(resource, blocks);
     }
     bridge.clearCache();
 
     // Detect
     for (InputFile inputFile : inputFiles) {
       Resource resource = mapping.createResource(inputFile.getFile(), fileSystem.getSourceDirs());
-      String resourceId = getFullKey(project, resource);
-      Collection<Block> fileBlocks = index.getByResourceId(resourceId);
+      String resourceKey = SonarEngine.getFullKey(project, resource);
+
+      Collection<Block> fileBlocks = index.getByResource(resource, resourceKey);
       List<CloneGroup> duplications = SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks);
       SonarEngine.save(context, resource, duplications);
     }
   }
 
-  private static String getFullKey(Project project, Resource resource) {
-    return new StringBuilder(ResourceModel.KEY_SIZE)
-        .append(project.getKey())
-        .append(':')
-        .append(resource.getKey())
-        .toString();
-  }
-
-  private int getMinimumTokens(Project project) {
-    Configuration conf = project.getConfiguration();
-    return conf.getInt("sonar.cpd." + project.getLanguageKey() + ".minimumTokens",
-        conf.getInt("sonar.cpd.minimumTokens", CoreProperties.CPD_MINIMUM_TOKENS_DEFAULT_VALUE));
-  }
-
   private CpdMapping getMapping(Language language) {
     if (mappings != null) {
       for (CpdMapping cpdMapping : mappings) {
diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java
index 2af3117d824..0c1a4345b61 100644
--- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java
@@ -20,17 +20,14 @@
 package org.sonar.plugins.cpd;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.SensorContext;
 import org.sonar.api.database.model.ResourceModel;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Measure;
 import org.sonar.api.resources.*;
 import org.sonar.api.utils.SonarException;
-import org.sonar.batch.index.ResourcePersister;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.block.BlockChunker;
 import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm;
@@ -42,8 +39,7 @@ import org.sonar.duplications.java.JavaTokenProducer;
 import org.sonar.duplications.statement.Statement;
 import org.sonar.duplications.statement.StatementChunker;
 import org.sonar.duplications.token.TokenChunker;
-import org.sonar.core.duplication.DuplicationDao;
-import org.sonar.plugins.cpd.index.DbDuplicationsIndex;
+import org.sonar.plugins.cpd.index.IndexFactory;
 import org.sonar.plugins.cpd.index.SonarDuplicationsIndex;
 
 import java.io.FileInputStream;
@@ -67,19 +63,10 @@ public class SonarEngine extends CpdEngine {
    */
   private static final int TIMEOUT = 5 * 60;
 
-  private final ResourcePersister resourcePersister;
-  private final DuplicationDao dao;
+  private final IndexFactory indexFactory;
 
-  /**
-   * For dry run, where is no access to database.
-   */
-  public SonarEngine() {
-    this(null, null);
-  }
-
-  public SonarEngine(ResourcePersister resourcePersister, DuplicationDao dao) {
-    this.resourcePersister = resourcePersister;
-    this.dao = dao;
+  public SonarEngine(IndexFactory indexFactory) {
+    this.indexFactory = indexFactory;
   }
 
   @Override
@@ -87,16 +74,7 @@ public class SonarEngine extends CpdEngine {
     return Java.INSTANCE.equals(language);
   }
 
-  /**
-   * @return true, if was enabled by user and database is available
-   */
-  private boolean isCrossProject(Project project) {
-    return project.getConfiguration().getBoolean(CoreProperties.CPD_CROSS_RPOJECT, CoreProperties.CPD_CROSS_RPOJECT_DEFAULT_VALUE)
-      && resourcePersister != null && dao != null
-      && StringUtils.isBlank(project.getConfiguration().getString(CoreProperties.PROJECT_BRANCH_PROPERTY));
-  }
-
-  private static String getFullKey(Project project, Resource resource) {
+  static String getFullKey(Project project, Resource resource) {
     return new StringBuilder(ResourceModel.KEY_SIZE)
       .append(project.getKey())
       .append(':')
@@ -112,14 +90,7 @@ public class SonarEngine extends CpdEngine {
     }
 
     // Create index
-    final SonarDuplicationsIndex index;
-    if (isCrossProject(project)) {
-      LOG.info("Cross-project analysis enabled");
-      index = new SonarDuplicationsIndex(new DbDuplicationsIndex(resourcePersister, project, dao));
-    } else {
-      LOG.info("Cross-project analysis disabled");
-      index = new SonarDuplicationsIndex();
-    }
+    final SonarDuplicationsIndex index = indexFactory.create(project);
 
     TokenChunker tokenChunker = JavaTokenProducer.build();
     StatementChunker statementChunker = JavaStatementBuilder.build();
diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/TokenizerBridge.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/TokenizerBridge.java
deleted file mode 100644
index 4d2760b1ab0..00000000000
--- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/TokenizerBridge.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Sonar, open source software quality management tool.
- * Copyright (C) 2008-2012 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * Sonar is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * Sonar is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with Sonar; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
- */
-package org.sonar.plugins.cpd;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-import net.sourceforge.pmd.cpd.SourceCode;
-import net.sourceforge.pmd.cpd.TokenEntry;
-import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.cpd.Tokens;
-import org.sonar.duplications.cpd.FileCodeLoaderWithoutCache;
-import org.sonar.duplications.statement.Statement;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-public class TokenizerBridge {
-
-  private final Tokenizer tokenizer;
-  private final String encoding;
-
-  public TokenizerBridge(Tokenizer tokenizer, String encoding) {
-    this.tokenizer = tokenizer;
-    this.encoding = encoding;
-    clearCache();
-  }
-
-  public List<Statement> tokenize(File file) {
-    SourceCode sourceCode = new SourceCode(new FileCodeLoaderWithoutCache(file, encoding));
-    Tokens tokens = new Tokens();
-    try {
-      tokenizer.tokenize(sourceCode, tokens);
-    } catch (IOException e) {
-      throw Throwables.propagate(e);
-    }
-    return convert(tokens.getTokens());
-  }
-
-  /**
-   * We expect that implementation of {@link Tokenizer} is correct:
-   * tokens ordered by occurrence in source code and last token is EOF.
-   */
-  private List<Statement> convert(List<TokenEntry> tokens) {
-    ImmutableList.Builder<Statement> result = ImmutableList.builder();
-    for (TokenEntry token : tokens) {
-      if (token != TokenEntry.EOF) {
-        int line = token.getBeginLine();
-        int id = token.getIdentifier();
-        result.add(new Statement(line, line, Integer.toString(id)));
-      }
-    }
-    return result.build();
-  }
-
-  public void clearCache() {
-    TokenEntry.clearImages();
-  }
-
-}
diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbDuplicationsIndex.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbDuplicationsIndex.java
index 6e9f3e43f68..23928299b90 100644
--- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbDuplicationsIndex.java
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbDuplicationsIndex.java
@@ -60,7 +60,8 @@ public class DbDuplicationsIndex {
 
   public void prepareCache(Resource resource) {
     int resourceSnapshotId = getSnapshotIdFor(resource);
-    List<DuplicationUnitDto> units = dao.selectCandidates(resourceSnapshotId, lastSnapshotId);
+    String languageKey = resource.getLanguage().getKey();
+    List<DuplicationUnitDto> units = dao.selectCandidates(resourceSnapshotId, lastSnapshotId, languageKey);
     cache.clear();
     // TODO Godin: maybe remove conversion of units to blocks?
     for (DuplicationUnitDto unit : units) {
diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/IndexFactory.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/IndexFactory.java
new file mode 100644
index 00000000000..3a0e3488170
--- /dev/null
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/IndexFactory.java
@@ -0,0 +1,69 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2012 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.plugins.cpd.index;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.BatchExtension;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.resources.Project;
+import org.sonar.batch.index.ResourcePersister;
+import org.sonar.core.duplication.DuplicationDao;
+
+public class IndexFactory implements BatchExtension {
+
+  private static final Logger LOG = LoggerFactory.getLogger(IndexFactory.class);
+
+  private final ResourcePersister resourcePersister;
+  private final DuplicationDao dao;
+
+  /**
+   * For dry run, where is no access to database.
+   */
+  public IndexFactory() {
+    this(null, null);
+  }
+
+  public IndexFactory(ResourcePersister resourcePersister, DuplicationDao dao) {
+    this.resourcePersister = resourcePersister;
+    this.dao = dao;
+  }
+
+  public SonarDuplicationsIndex create(Project project) {
+    if (isCrossProject(project)) {
+      LOG.info("Cross-project analysis enabled");
+      return new SonarDuplicationsIndex(new DbDuplicationsIndex(resourcePersister, project, dao));
+    } else {
+      LOG.info("Cross-project analysis disabled");
+      return new SonarDuplicationsIndex();
+    }
+  }
+
+  /**
+   * @return true, if was enabled by user and database is available
+   */
+  private boolean isCrossProject(Project project) {
+    return project.getConfiguration().getBoolean(CoreProperties.CPD_CROSS_RPOJECT, CoreProperties.CPD_CROSS_RPOJECT_DEFAULT_VALUE)
+      && resourcePersister != null && dao != null
+      && StringUtils.isBlank(project.getConfiguration().getString(CoreProperties.PROJECT_BRANCH_PROPERTY));
+  }
+
+}
diff --git a/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java
index 95ef890ac04..f78c6e5b76b 100644
--- a/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java
+++ b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java
@@ -19,15 +19,16 @@
  */
 package org.sonar.plugins.cpd;
 
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.junit.Test;
 import org.sonar.api.batch.CpdMapping;
 import org.sonar.api.resources.Project;
+import org.sonar.plugins.cpd.index.IndexFactory;
+
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertTrue;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
 
 public class CpdSensorTest {
 
@@ -38,7 +39,7 @@ public class CpdSensorTest {
 
     Project project = createJavaProject().setConfiguration(conf);
 
-    CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0]));
+    CpdSensor sensor = new CpdSensor(new SonarEngine(new IndexFactory()), new PmdEngine(new CpdMapping[0]));
     assertTrue(sensor.isSkipped(project));
   }
 
@@ -46,7 +47,7 @@ public class CpdSensorTest {
   public void doNotSkipByDefault() {
     Project project = createJavaProject().setConfiguration(new PropertiesConfiguration());
 
-    CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0]));
+    CpdSensor sensor = new CpdSensor(new SonarEngine(new IndexFactory()), new PmdEngine(new CpdMapping[0]));
     assertFalse(sensor.isSkipped(project));
   }
 
@@ -59,7 +60,7 @@ public class CpdSensorTest {
     Project phpProject = createPhpProject().setConfiguration(conf);
     Project javaProject = createJavaProject().setConfiguration(conf);
 
-    CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0]));
+    CpdSensor sensor = new CpdSensor(new SonarEngine(new IndexFactory()), new PmdEngine(new CpdMapping[0]));
     assertTrue(sensor.isSkipped(phpProject));
     assertFalse(sensor.isSkipped(javaProject));
   }
@@ -68,7 +69,7 @@ public class CpdSensorTest {
   public void engine() {
     PropertiesConfiguration conf = new PropertiesConfiguration();
     Project project = createJavaProject().setConfiguration(conf);
-    CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0]));
+    CpdSensor sensor = new CpdSensor(new SonarEngine(new IndexFactory()), new PmdEngine(new CpdMapping[0]));
 
     assertThat(sensor.isSonarEngineEnabled(project), is(true));
     conf.setProperty("sonar.cpd.engine", "pmd");