SONAR-1091 Add CPD over different projects

* Add table clone_blocks

* Add DbCloneIndex, which can be activated in sonar-cpd-plugin using
  property "sonar.cpd.cross_project=true"

7 files changed, 322 insertions, 5 deletions

diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java
index b992598b6f8..b3cd865cfc2 100644
--- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java
@@ -28,6 +28,9 @@ import java.util.List;
 import java.util.Set;
 
 import org.sonar.api.batch.SensorContext;
+import org.sonar.api.database.DatabaseSession;
+import org.sonar.api.database.model.ResourceModel;
+import org.sonar.api.database.model.Snapshot;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Measure;
 import org.sonar.api.resources.InputFile;
@@ -36,6 +39,8 @@ import org.sonar.api.resources.JavaFile;
 import org.sonar.api.resources.Language;
 import org.sonar.api.resources.Project;
 import org.sonar.api.resources.Resource;
+import org.sonar.api.utils.Logs;
+import org.sonar.batch.index.ResourcePersister;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.block.BlockChunker;
 import org.sonar.duplications.detector.original.OriginalCloneDetectionAlgorithm;
@@ -49,6 +54,8 @@ import org.sonar.duplications.statement.Statement;
 import org.sonar.duplications.statement.StatementChunker;
 import org.sonar.duplications.token.TokenChunker;
 import org.sonar.duplications.token.TokenQueue;
+import org.sonar.plugins.cpd.index.CombinedCloneIndex;
+import org.sonar.plugins.cpd.index.DbCloneIndex;
 
 import com.google.common.collect.Lists;
 
@@ -56,10 +63,30 @@ public class SonarEngine implements CpdEngine {
 
   private static final int BLOCK_SIZE = 13;
 
+  private final ResourcePersister resourcePersister;
+  private final DatabaseSession dbSession;
+
+  public SonarEngine(ResourcePersister resourcePersister, DatabaseSession dbSession) {
+    this.resourcePersister = resourcePersister;
+    this.dbSession = dbSession;
+  }
+
   public boolean isLanguageSupported(Language language) {
     return Java.INSTANCE.equals(language);
   }
 
+  private static boolean isCrossProject(Project project) {
+    return project.getConfiguration().getBoolean("sonar.cpd.cross_project", false);
+  }
+
+  private static String getFullKey(Project project, Resource resource) {
+    return new StringBuilder(ResourceModel.KEY_SIZE)
+        .append(project.getKey())
+        .append(':')
+        .append(resource.getKey())
+        .toString();
+  }
+
   public void analyse(Project project, SensorContext context) {
     List<InputFile> inputFiles = project.getFileSystem().mainFiles(project.getLanguageKey());
     if (inputFiles.isEmpty()) {
@@ -68,6 +95,13 @@ public class SonarEngine implements CpdEngine {
 
     // Create index
     CloneIndex index = new PackedMemoryCloneIndex();
+    if (isCrossProject(project)) {
+      Logs.INFO.info("Enabled cross-project analysis");
+      Snapshot currentSnapshot = resourcePersister.getSnapshot(project);
+      Snapshot lastSnapshot = resourcePersister.getLastSnapshot(currentSnapshot, false);
+      DbCloneIndex db = new DbCloneIndex(dbSession, currentSnapshot.getId(), lastSnapshot == null ? null : lastSnapshot.getId());
+      index = new CombinedCloneIndex(index, db);
+    }
 
     TokenChunker tokenChunker = JavaTokenProducer.build();
     StatementChunker statementChunker = JavaStatementBuilder.build();
@@ -78,7 +112,7 @@ public class SonarEngine implements CpdEngine {
       TokenQueue tokenQueue = tokenChunker.chunk(file);
       List<Statement> statements = statementChunker.chunk(tokenQueue);
       Resource resource = getResource(inputFile);
-      List<Block> blocks = blockChunker.chunk(resource.getKey(), statements);
+      List<Block> blocks = blockChunker.chunk(getFullKey(project, resource), statements);
       for (Block block : blocks) {
         index.insert(block);
       }
@@ -88,7 +122,7 @@ public class SonarEngine implements CpdEngine {
     for (InputFile inputFile : inputFiles) {
       Resource resource = getResource(inputFile);
 
-      List<Block> fileBlocks = Lists.newArrayList(index.getByResourceId(resource.getKey()));
+      List<Block> fileBlocks = Lists.newArrayList(index.getByResourceId(getFullKey(project, resource)));
       List<CloneGroup> clones = OriginalCloneDetectionAlgorithm.detect(index, fileBlocks);
       if (!clones.isEmpty()) {
         // Save
diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/CombinedCloneIndex.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/CombinedCloneIndex.java
new file mode 100644
index 00000000000..0b2f4e4c929
--- /dev/null
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/CombinedCloneIndex.java
@@ -0,0 +1,59 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2011 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.plugins.cpd.index;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.sonar.duplications.block.Block;
+import org.sonar.duplications.block.ByteArray;
+import org.sonar.duplications.index.AbstractCloneIndex;
+import org.sonar.duplications.index.CloneIndex;
+
+import com.google.common.collect.Lists;
+
+public class CombinedCloneIndex extends AbstractCloneIndex {
+
+  private final CloneIndex mem;
+  private final DbCloneIndex db;
+
+  public CombinedCloneIndex(CloneIndex mem, DbCloneIndex db) {
+    this.mem = mem;
+    this.db = db;
+  }
+
+  public Collection<Block> getByResourceId(String resourceId) {
+    db.prepareCache(resourceId);
+    return mem.getByResourceId(resourceId);
+  }
+
+  public Collection<Block> getBySequenceHash(ByteArray hash) {
+    List<Block> result = Lists.newArrayList();
+    result.addAll(mem.getBySequenceHash(hash));
+    result.addAll(db.getBySequenceHash(hash));
+    return result;
+  }
+
+  public void insert(Block block) {
+    mem.insert(block);
+    db.insert(block);
+  }
+
+}
diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbCloneIndex.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbCloneIndex.java
new file mode 100644
index 00000000000..06f6c481cad
--- /dev/null
+++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbCloneIndex.java
@@ -0,0 +1,101 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2011 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.plugins.cpd.index;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.sonar.api.database.DatabaseSession;
+import org.sonar.duplications.block.Block;
+import org.sonar.duplications.block.ByteArray;
+import org.sonar.duplications.index.AbstractCloneIndex;
+import org.sonar.jpa.entity.CloneBlock;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public class DbCloneIndex extends AbstractCloneIndex {
+
+  private final Map<ByteArray, List<Block>> cache = Maps.newHashMap();
+
+  private DatabaseSession session;
+  private int currentSnapshotId;
+  private Integer lastSnapshotId;
+
+  public DbCloneIndex(DatabaseSession session, Integer currentSnapshotId, Integer lastSnapshotId) {
+    this.session = session;
+    this.currentSnapshotId = currentSnapshotId;
+    this.lastSnapshotId = lastSnapshotId;
+  }
+
+  public void prepareCache(String resourceKey) {
+    String sql = "SELECT block.id, hash, block.snapshot_id, resource_key, index_in_file, start_line, end_line FROM clone_blocks AS block, snapshots AS snapshot" +
+        " WHERE block.snapshot_id=snapshot.id AND snapshot.islast=true" +
+        " AND hash IN ( SELECT hash FROM clone_blocks WHERE resource_key = :resource_key AND snapshot_id = :current_snapshot_id )";
+    if (lastSnapshotId != null) {
+      // Filter for blocks from previous snapshot of current project
+      sql += " AND snapshot.id != " + lastSnapshotId;
+    }
+    List<CloneBlock> blocks = session.getEntityManager()
+        .createNativeQuery(sql, CloneBlock.class)
+        .setParameter("resource_key", resourceKey)
+        .setParameter("current_snapshot_id", currentSnapshotId)
+        .getResultList();
+
+    cache.clear();
+    for (CloneBlock dbBlock : blocks) {
+      Block block = new Block(dbBlock.getResourceKey(), new ByteArray(dbBlock.getHash()), dbBlock.getIndexInFile(), dbBlock.getStartLine(), dbBlock.getEndLine());
+
+      List<Block> sameHash = cache.get(block.getBlockHash());
+      if (sameHash == null) {
+        sameHash = Lists.newArrayList();
+        cache.put(block.getBlockHash(), sameHash);
+      }
+      sameHash.add(block);
+    }
+  }
+
+  public Collection<Block> getByResourceId(String resourceId) {
+    throw new UnsupportedOperationException();
+  }
+
+  public Collection<Block> getBySequenceHash(ByteArray sequenceHash) {
+    List<Block> result = cache.get(sequenceHash);
+    if (result != null) {
+      return result;
+    } else {
+      // not in cache
+      return Collections.emptyList();
+    }
+  }
+
+  public void insert(Block block) {
+    CloneBlock dbBlock = new CloneBlock(currentSnapshotId,
+        block.getBlockHash().toString(),
+        block.getResourceId(),
+        block.getIndexInFile(),
+        block.getFirstLineNumber(),
+        block.getLastLineNumber());
+    session.save(dbBlock);
+  }
+
+}
diff --git a/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java
index 42c1f59d3a4..1ff0bee17ce 100644
--- a/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java
+++ b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java
@@ -38,7 +38,7 @@ public class CpdSensorTest {
 
     Project project = createJavaProject().setConfiguration(conf);
 
-    CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0]));
+    CpdSensor sensor = new CpdSensor(new SonarEngine(null, null), new PmdEngine(new CpdMapping[0]));
     assertTrue(sensor.isSkipped(project));
   }
 
@@ -46,7 +46,7 @@ public class CpdSensorTest {
   public void doNotSkipByDefault() {
     Project project = createJavaProject().setConfiguration(new PropertiesConfiguration());
 
-    CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0]));
+    CpdSensor sensor = new CpdSensor(new SonarEngine(null, null), new PmdEngine(new CpdMapping[0]));
     assertFalse(sensor.isSkipped(project));
   }
 
@@ -59,7 +59,7 @@ public class CpdSensorTest {
     Project phpProject = createPhpProject().setConfiguration(conf);
     Project javaProject = createJavaProject().setConfiguration(conf);
 
-    CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0]));
+    CpdSensor sensor = new CpdSensor(new SonarEngine(null, null), new PmdEngine(new CpdMapping[0]));
     assertTrue(sensor.isSkipped(phpProject));
     assertFalse(sensor.isSkipped(javaProject));
   }
diff --git a/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/index/DbCloneIndexTest.java b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/index/DbCloneIndexTest.java
new file mode 100644
index 00000000000..86698e89d64
--- /dev/null
+++ b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/index/DbCloneIndexTest.java
@@ -0,0 +1,71 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2011 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.plugins.cpd.index;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.util.Collection;
+import java.util.Iterator;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.sonar.duplications.block.Block;
+import org.sonar.duplications.block.ByteArray;
+import org.sonar.jpa.test.AbstractDbUnitTestCase;
+
+public class DbCloneIndexTest extends AbstractDbUnitTestCase {
+
+  private DbCloneIndex index;
+
+  @Before
+  public void setUp() {
+    index = new DbCloneIndex(getSession(), 5, 4);
+  }
+
+  @Test(expected = UnsupportedOperationException.class)
+  public void shouldNotGetByResource() {
+    index.getByResourceId("foo");
+  }
+
+  @Test
+  public void shouldGetByHash() {
+    setupData("fixture");
+
+    index.prepareCache("foo");
+    Collection<Block> blocks = index.getBySequenceHash(new ByteArray("aa"));
+    Iterator<Block> blocksIterator = blocks.iterator();
+
+    assertThat(blocks.size(), is(1));
+
+    Block block = blocksIterator.next();
+    assertThat(block.getResourceId(), is("bar-last"));
+  }
+
+  @Test
+  public void shouldInsert() {
+    setupData("fixture");
+
+    index.insert(new Block("baz", new ByteArray("bb"), 0, 0, 1));
+
+    checkTables("shouldInsert", "clone_blocks");
+  }
+
+}
diff --git a/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/fixture.xml b/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/fixture.xml
new file mode 100644
index 00000000000..95599894c1b
--- /dev/null
+++ b/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/fixture.xml
@@ -0,0 +1,25 @@
+<dataset>
+
+  <snapshots id="1" status="P" islast="false" />
+  <snapshots id="2" status="P" islast="true" />
+
+  <snapshots id="3" status="P" islast="false" />
+  <snapshots id="4" status="P" islast="true" />
+  <snapshots id="5" status="U" islast="false" />
+
+  <!-- Old snapshot of another project -->
+  <clone_blocks id="1" snapshot_id="1" hash="aa" resource_key="bar-old" index_in_file="0" start_line="0" end_line="1" />
+
+  <!-- Last snapshot of another project -->
+  <clone_blocks id="2" snapshot_id="2" hash="aa" resource_key="bar-last" index_in_file="0" start_line="0" end_line="1" />
+
+  <!-- Old snapshot of current project -->
+  <clone_blocks id="3" snapshot_id="3" hash="aa" resource_key="foo-old" index_in_file="0" start_line="0" end_line="1" />
+
+  <!-- Last snapshot of current project -->
+  <clone_blocks id="4" snapshot_id="4" hash="aa" resource_key="foo-last" index_in_file="0" start_line="0" end_line="1" />
+
+  <!-- New snapshot of current project -->
+  <clone_blocks id="5" snapshot_id="5" hash="aa" resource_key="foo" index_in_file="0" start_line="0" end_line="1" />
+
+</dataset>
diff --git a/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/shouldInsert-result.xml b/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/shouldInsert-result.xml
new file mode 100644
index 00000000000..ae2767dfedb
--- /dev/null
+++ b/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/shouldInsert-result.xml
@@ -0,0 +1,27 @@
+<dataset>
+
+  <snapshots id="1" status="P" islast="false" />
+  <snapshots id="2" status="P" islast="true" />
+
+  <snapshots id="3" status="P" islast="false" />
+  <snapshots id="4" status="P" islast="true" />
+  <snapshots id="5" status="U" islast="false" />
+
+  <!-- Old snapshot of another project -->
+  <clone_blocks id="1" snapshot_id="1" hash="aa" resource_key="bar-old" index_in_file="0" start_line="0" end_line="1" />
+
+  <!-- Last snapshot of another project -->
+  <clone_blocks id="2" snapshot_id="2" hash="aa" resource_key="bar-last" index_in_file="0" start_line="0" end_line="1" />
+
+  <!-- Old snapshot of current project -->
+  <clone_blocks id="3" snapshot_id="3" hash="aa" resource_key="foo-old" index_in_file="0" start_line="0" end_line="1" />
+
+  <!-- Last snapshot of current project -->
+  <clone_blocks id="4" snapshot_id="4" hash="aa" resource_key="foo-last" index_in_file="0" start_line="0" end_line="1" />
+
+  <!-- New snapshot of current project -->
+  <clone_blocks id="5" snapshot_id="5" hash="aa" resource_key="foo" index_in_file="0" start_line="0" end_line="1" />
+
+  <clone_blocks id="6" snapshot_id="5" hash="bb" resource_key="baz" index_in_file="0" start_line="0" end_line="1" />
+
+</dataset>