SONAR-3181,SONAR-3139 Allow filtering by number of tokens

To do this each block stores two additional integers - index for first
and last token. So total size of in-memory storage has been increased.

3 files changed, 20 insertions, 15 deletions

diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/PmdEngine.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/PmdEngine.java
index 08b590c0917..e84f22e36de 100644
--- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/PmdEngine.java
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/PmdEngine.java
@@ -19,12 +19,9 @@
  */
 package org.sonar.plugins.cpd;
 
-import java.io.IOException;
-import java.nio.charset.Charset;
-
+import com.google.common.annotations.VisibleForTesting;
 import net.sourceforge.pmd.cpd.AbstractLanguage;
 import net.sourceforge.pmd.cpd.TokenEntry;
-
 import org.apache.commons.configuration.Configuration;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.CpdMapping;
@@ -33,7 +30,8 @@ import org.sonar.api.resources.Language;
 import org.sonar.api.resources.Project;
 import org.sonar.duplications.cpd.CPD;
 
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.nio.charset.Charset;
 
 public class PmdEngine extends CpdEngine {
 
@@ -103,7 +101,7 @@ public class PmdEngine extends CpdEngine {
   }
 
   @VisibleForTesting
-  int getMinimumTokens(Project project) {
+  static int getMinimumTokens(Project project) {
     Configuration conf = project.getConfiguration();
     return conf.getInt("sonar.cpd." + project.getLanguageKey() + ".minimumTokens",
         conf.getInt("sonar.cpd.minimumTokens", CoreProperties.CPD_MINIMUM_TOKENS_DEFAULT_VALUE));
diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarBridgeEngine.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarBridgeEngine.java
index a1cab7980d3..2e736acde3f 100644
--- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarBridgeEngine.java
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarBridgeEngine.java
@@ -19,11 +19,12 @@
  */
 package org.sonar.plugins.cpd;
 
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
 import org.sonar.api.batch.CpdMapping;
 import org.sonar.api.batch.SensorContext;
 import org.sonar.api.resources.*;
 import org.sonar.duplications.block.Block;
-import org.sonar.duplications.block.BlockChunker;
 import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm;
 import org.sonar.duplications.index.CloneGroup;
 import org.sonar.duplications.internal.pmd.TokenizerBridge;
@@ -35,8 +36,6 @@ import java.util.List;
 
 public class SonarBridgeEngine extends CpdEngine {
 
-  private static final int BLOCK_SIZE = 10;
-
   private final IndexFactory indexFactory;
   private final CpdMapping[] mappings;
 
@@ -67,24 +66,32 @@ public class SonarBridgeEngine extends CpdEngine {
     // Create index
     SonarDuplicationsIndex index = indexFactory.create(project);
 
-    BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE);
     TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), fileSystem.getSourceCharset().name());
     for (InputFile inputFile : inputFiles) {
       Resource resource = mapping.createResource(inputFile.getFile(), fileSystem.getSourceDirs());
       String resourceId = SonarEngine.getFullKey(project, resource);
-      List<Block> blocks = blockChunker.chunk(resourceId, bridge.tokenize(inputFile.getFile()));
+      List<Block> blocks = bridge.chunk(resourceId, inputFile.getFile());
       index.insert(resource, blocks);
     }
-    bridge.clearCache();
 
     // Detect
+    final int minimumTokens = PmdEngine.getMinimumTokens(project);
+    Predicate<CloneGroup> minimumTokensPredicate = new Predicate<CloneGroup>() {
+      public boolean apply(CloneGroup input) {
+        return input.getLengthInUnits() >= minimumTokens;
+      }
+    };
+
     for (InputFile inputFile : inputFiles) {
       Resource resource = mapping.createResource(inputFile.getFile(), fileSystem.getSourceDirs());
       String resourceKey = SonarEngine.getFullKey(project, resource);
 
       Collection<Block> fileBlocks = index.getByResource(resource, resourceKey);
       List<CloneGroup> duplications = SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks);
-      SonarEngine.save(context, resource, duplications);
+
+      Iterable<CloneGroup> filtered = Iterables.filter(duplications, minimumTokensPredicate);
+
+      SonarEngine.save(context, resource, filtered);
     }
   }
 
diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java
index 0c1a4345b61..9ae22f814c3 100644
--- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java
@@ -164,8 +164,8 @@ public class SonarEngine extends CpdEngine {
     return JavaFile.fromRelativePath(inputFile.getRelativePath(), false);
   }
 
-  static void save(SensorContext context, Resource resource, List<CloneGroup> clones) {
-    if (clones == null || clones.isEmpty()) {
+  static void save(SensorContext context, Resource resource, Iterable<CloneGroup> clones) {
+    if (clones == null || !clones.iterator().hasNext()) {
       return;
     }
     // Calculate number of lines and blocks