SONAR-3752 Improve detection of duplications

Multiple successive and identical lines should be treated as one monolitic fragment.

4 files changed, 54 insertions, 7 deletions

diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/block/BlockChunker.java b/sonar-duplications/src/main/java/org/sonar/duplications/block/BlockChunker.java
index 0851eb21f04..40779fed35a 100644
--- a/sonar-duplications/src/main/java/org/sonar/duplications/block/BlockChunker.java
+++ b/sonar-duplications/src/main/java/org/sonar/duplications/block/BlockChunker.java
@@ -55,6 +55,24 @@ public class BlockChunker {
   }
 
   public List<Block> chunk(String resourceId, List<Statement> statements) {
+    List<Statement> filtered = Lists.newArrayList();
+    int i = 0;
+    while (i < statements.size()) {
+      Statement first = statements.get(i);
+      int j = i + 1;
+      while (j < statements.size() && statements.get(j).getValue().equals(first.getValue())) {
+        j++;
+      }
+      if (i < j - 1) {
+        Statement last = statements.get(j - 1);
+        filtered.add(new Statement(first.getStartLine(), last.getEndLine(), first.getValue()));
+      } else {
+        filtered.add(statements.get(i));
+      }
+      i = j;
+    }
+    statements = filtered;
+
     if (statements.size() < blockSize) {
       return Collections.emptyList();
     }
diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/PmdBlockChunker.java b/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/PmdBlockChunker.java
index 7fb3652adc0..1c8e9624f13 100644
--- a/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/PmdBlockChunker.java
+++ b/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/PmdBlockChunker.java
@@ -49,6 +49,24 @@ public class PmdBlockChunker {
   }
 
   public List<Block> chunk(String resourceId, List<TokensLine> fragments) {
+    List<TokensLine> filtered = Lists.newArrayList();
+    int i = 0;
+    while (i < fragments.size()) {
+      TokensLine first = fragments.get(i);
+      int j = i + 1;
+      while (j < fragments.size() && fragments.get(j).getValue().equals(first.getValue())) {
+        j++;
+      }
+      if (i < j - 1) {
+        TokensLine last = fragments.get(j - 1);
+        filtered.add(new TokensLine(first.getStartUnit(), last.getEndUnit(), first.getStartLine(), last.getEndLine(), first.getValue()));
+      } else {
+        filtered.add(fragments.get(i));
+      }
+      i = j;
+    }
+    fragments = filtered;
+
     if (fragments.size() < blockSize) {
       return Collections.emptyList();
     }
diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokenizerBridge.java b/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokenizerBridge.java
index 7881ee60cfd..0dfc312ff94 100644
--- a/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokenizerBridge.java
+++ b/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokenizerBridge.java
@@ -94,7 +94,7 @@ public class TokenizerBridge {
 
   private static void addNewTokensLine(ImmutableList.Builder<TokensLine> result, int startUnit, int endUnit, int startLine, StringBuilder sb) {
     if (sb.length() != 0) {
-      result.add(new TokensLine(startUnit, endUnit, startLine, sb.toString().hashCode()));
+      result.add(new TokensLine(startUnit, endUnit, startLine, sb.toString()));
       sb.setLength(0);
     }
   }
diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokensLine.java b/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokensLine.java
index 2f832bb50af..44fcaafb1c3 100644
--- a/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokensLine.java
+++ b/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokensLine.java
@@ -27,31 +27,42 @@ import org.sonar.duplications.CodeFragment;
  */
 class TokensLine implements CodeFragment {
 
+  private final String value;
+
   private final int startLine;
+  private final int endLine;
   private final int hashCode;
 
   private final int startUnit;
   private final int endUnit;
 
-  public TokensLine(int startUnit, int endUnit, int startLine, int hashCode) {
+
+  public TokensLine(int startUnit, int endUnit, int startLine, String value) {
+    this(startUnit, endUnit, startLine, startLine, value);
+  }
+
+  public TokensLine(int startUnit, int endUnit, int startLine, int endLine, String value) {
     Preconditions.checkArgument(startLine > 0);
     // TODO do we have requirements for length and hashcode ?
     this.startLine = startLine;
-    this.hashCode = hashCode;
+    this.endLine = endLine;
+    this.value = value;
+    this.hashCode = value.hashCode();
 
     this.startUnit = startUnit;
     this.endUnit = endUnit;
   }
 
+  public String getValue() {
+    return value;
+  }
+
   public int getStartLine() {
     return startLine;
   }
 
-  /**
-   * Same as {@link #getStartLine()}
-   */
   public int getEndLine() {
-    return startLine;
+    return endLine;
   }
 
   public int getHashCode() {