From 7cf051c0f7d8ec017381b05c85044e9321c6c4f7 Mon Sep 17 00:00:00 2001 From: Evgeny Mandrikov Date: Wed, 31 Aug 2011 19:27:13 +0400 Subject: SONAR-1091 Add CPD over different projects * Add table clone_blocks * Add DbCloneIndex, which can be activated in sonar-cpd-plugin using property "sonar.cpd.cross_project=true" --- .../java/org/sonar/plugins/cpd/SonarEngine.java | 38 +++++++- .../plugins/cpd/index/CombinedCloneIndex.java | 59 ++++++++++++ .../org/sonar/plugins/cpd/index/DbCloneIndex.java | 101 +++++++++++++++++++++ .../java/org/sonar/plugins/cpd/CpdSensorTest.java | 6 +- .../sonar/plugins/cpd/index/DbCloneIndexTest.java | 71 +++++++++++++++ .../plugins/cpd/index/DbCloneIndexTest/fixture.xml | 25 +++++ .../index/DbCloneIndexTest/shouldInsert-result.xml | 27 ++++++ 7 files changed, 322 insertions(+), 5 deletions(-) create mode 100644 plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/CombinedCloneIndex.java create mode 100644 plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbCloneIndex.java create mode 100644 plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/index/DbCloneIndexTest.java create mode 100644 plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/fixture.xml create mode 100644 plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/shouldInsert-result.xml (limited to 'plugins/sonar-cpd-plugin/src') diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java index b992598b6f8..b3cd865cfc2 100644 --- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java +++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/SonarEngine.java @@ -28,6 +28,9 @@ import java.util.List; import java.util.Set; import org.sonar.api.batch.SensorContext; +import org.sonar.api.database.DatabaseSession; +import org.sonar.api.database.model.ResourceModel; +import org.sonar.api.database.model.Snapshot; import org.sonar.api.measures.CoreMetrics; import org.sonar.api.measures.Measure; import org.sonar.api.resources.InputFile; @@ -36,6 +39,8 @@ import org.sonar.api.resources.JavaFile; import org.sonar.api.resources.Language; import org.sonar.api.resources.Project; import org.sonar.api.resources.Resource; +import org.sonar.api.utils.Logs; +import org.sonar.batch.index.ResourcePersister; import org.sonar.duplications.block.Block; import org.sonar.duplications.block.BlockChunker; import org.sonar.duplications.detector.original.OriginalCloneDetectionAlgorithm; @@ -49,6 +54,8 @@ import org.sonar.duplications.statement.Statement; import org.sonar.duplications.statement.StatementChunker; import org.sonar.duplications.token.TokenChunker; import org.sonar.duplications.token.TokenQueue; +import org.sonar.plugins.cpd.index.CombinedCloneIndex; +import org.sonar.plugins.cpd.index.DbCloneIndex; import com.google.common.collect.Lists; @@ -56,10 +63,30 @@ public class SonarEngine implements CpdEngine { private static final int BLOCK_SIZE = 13; + private final ResourcePersister resourcePersister; + private final DatabaseSession dbSession; + + public SonarEngine(ResourcePersister resourcePersister, DatabaseSession dbSession) { + this.resourcePersister = resourcePersister; + this.dbSession = dbSession; + } + public boolean isLanguageSupported(Language language) { return Java.INSTANCE.equals(language); } + private static boolean isCrossProject(Project project) { + return project.getConfiguration().getBoolean("sonar.cpd.cross_project", false); + } + + private static String getFullKey(Project project, Resource resource) { + return new StringBuilder(ResourceModel.KEY_SIZE) + .append(project.getKey()) + .append(':') + .append(resource.getKey()) + .toString(); + } + public void analyse(Project project, SensorContext context) { List inputFiles = project.getFileSystem().mainFiles(project.getLanguageKey()); if (inputFiles.isEmpty()) { @@ -68,6 +95,13 @@ public class SonarEngine implements CpdEngine { // Create index CloneIndex index = new PackedMemoryCloneIndex(); + if (isCrossProject(project)) { + Logs.INFO.info("Enabled cross-project analysis"); + Snapshot currentSnapshot = resourcePersister.getSnapshot(project); + Snapshot lastSnapshot = resourcePersister.getLastSnapshot(currentSnapshot, false); + DbCloneIndex db = new DbCloneIndex(dbSession, currentSnapshot.getId(), lastSnapshot == null ? null : lastSnapshot.getId()); + index = new CombinedCloneIndex(index, db); + } TokenChunker tokenChunker = JavaTokenProducer.build(); StatementChunker statementChunker = JavaStatementBuilder.build(); @@ -78,7 +112,7 @@ public class SonarEngine implements CpdEngine { TokenQueue tokenQueue = tokenChunker.chunk(file); List statements = statementChunker.chunk(tokenQueue); Resource resource = getResource(inputFile); - List blocks = blockChunker.chunk(resource.getKey(), statements); + List blocks = blockChunker.chunk(getFullKey(project, resource), statements); for (Block block : blocks) { index.insert(block); } @@ -88,7 +122,7 @@ public class SonarEngine implements CpdEngine { for (InputFile inputFile : inputFiles) { Resource resource = getResource(inputFile); - List fileBlocks = Lists.newArrayList(index.getByResourceId(resource.getKey())); + List fileBlocks = Lists.newArrayList(index.getByResourceId(getFullKey(project, resource))); List clones = OriginalCloneDetectionAlgorithm.detect(index, fileBlocks); if (!clones.isEmpty()) { // Save diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/CombinedCloneIndex.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/CombinedCloneIndex.java new file mode 100644 index 00000000000..0b2f4e4c929 --- /dev/null +++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/CombinedCloneIndex.java @@ -0,0 +1,59 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2011 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.plugins.cpd.index; + +import java.util.Collection; +import java.util.List; + +import org.sonar.duplications.block.Block; +import org.sonar.duplications.block.ByteArray; +import org.sonar.duplications.index.AbstractCloneIndex; +import org.sonar.duplications.index.CloneIndex; + +import com.google.common.collect.Lists; + +public class CombinedCloneIndex extends AbstractCloneIndex { + + private final CloneIndex mem; + private final DbCloneIndex db; + + public CombinedCloneIndex(CloneIndex mem, DbCloneIndex db) { + this.mem = mem; + this.db = db; + } + + public Collection getByResourceId(String resourceId) { + db.prepareCache(resourceId); + return mem.getByResourceId(resourceId); + } + + public Collection getBySequenceHash(ByteArray hash) { + List result = Lists.newArrayList(); + result.addAll(mem.getBySequenceHash(hash)); + result.addAll(db.getBySequenceHash(hash)); + return result; + } + + public void insert(Block block) { + mem.insert(block); + db.insert(block); + } + +} diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbCloneIndex.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbCloneIndex.java new file mode 100644 index 00000000000..06f6c481cad --- /dev/null +++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/index/DbCloneIndex.java @@ -0,0 +1,101 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2011 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.plugins.cpd.index; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.sonar.api.database.DatabaseSession; +import org.sonar.duplications.block.Block; +import org.sonar.duplications.block.ByteArray; +import org.sonar.duplications.index.AbstractCloneIndex; +import org.sonar.jpa.entity.CloneBlock; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +public class DbCloneIndex extends AbstractCloneIndex { + + private final Map> cache = Maps.newHashMap(); + + private DatabaseSession session; + private int currentSnapshotId; + private Integer lastSnapshotId; + + public DbCloneIndex(DatabaseSession session, Integer currentSnapshotId, Integer lastSnapshotId) { + this.session = session; + this.currentSnapshotId = currentSnapshotId; + this.lastSnapshotId = lastSnapshotId; + } + + public void prepareCache(String resourceKey) { + String sql = "SELECT block.id, hash, block.snapshot_id, resource_key, index_in_file, start_line, end_line FROM clone_blocks AS block, snapshots AS snapshot" + + " WHERE block.snapshot_id=snapshot.id AND snapshot.islast=true" + + " AND hash IN ( SELECT hash FROM clone_blocks WHERE resource_key = :resource_key AND snapshot_id = :current_snapshot_id )"; + if (lastSnapshotId != null) { + // Filter for blocks from previous snapshot of current project + sql += " AND snapshot.id != " + lastSnapshotId; + } + List blocks = session.getEntityManager() + .createNativeQuery(sql, CloneBlock.class) + .setParameter("resource_key", resourceKey) + .setParameter("current_snapshot_id", currentSnapshotId) + .getResultList(); + + cache.clear(); + for (CloneBlock dbBlock : blocks) { + Block block = new Block(dbBlock.getResourceKey(), new ByteArray(dbBlock.getHash()), dbBlock.getIndexInFile(), dbBlock.getStartLine(), dbBlock.getEndLine()); + + List sameHash = cache.get(block.getBlockHash()); + if (sameHash == null) { + sameHash = Lists.newArrayList(); + cache.put(block.getBlockHash(), sameHash); + } + sameHash.add(block); + } + } + + public Collection getByResourceId(String resourceId) { + throw new UnsupportedOperationException(); + } + + public Collection getBySequenceHash(ByteArray sequenceHash) { + List result = cache.get(sequenceHash); + if (result != null) { + return result; + } else { + // not in cache + return Collections.emptyList(); + } + } + + public void insert(Block block) { + CloneBlock dbBlock = new CloneBlock(currentSnapshotId, + block.getBlockHash().toString(), + block.getResourceId(), + block.getIndexInFile(), + block.getFirstLineNumber(), + block.getLastLineNumber()); + session.save(dbBlock); + } + +} diff --git a/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java index 42c1f59d3a4..1ff0bee17ce 100644 --- a/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java +++ b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/CpdSensorTest.java @@ -38,7 +38,7 @@ public class CpdSensorTest { Project project = createJavaProject().setConfiguration(conf); - CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0])); + CpdSensor sensor = new CpdSensor(new SonarEngine(null, null), new PmdEngine(new CpdMapping[0])); assertTrue(sensor.isSkipped(project)); } @@ -46,7 +46,7 @@ public class CpdSensorTest { public void doNotSkipByDefault() { Project project = createJavaProject().setConfiguration(new PropertiesConfiguration()); - CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0])); + CpdSensor sensor = new CpdSensor(new SonarEngine(null, null), new PmdEngine(new CpdMapping[0])); assertFalse(sensor.isSkipped(project)); } @@ -59,7 +59,7 @@ public class CpdSensorTest { Project phpProject = createPhpProject().setConfiguration(conf); Project javaProject = createJavaProject().setConfiguration(conf); - CpdSensor sensor = new CpdSensor(new SonarEngine(), new PmdEngine(new CpdMapping[0])); + CpdSensor sensor = new CpdSensor(new SonarEngine(null, null), new PmdEngine(new CpdMapping[0])); assertTrue(sensor.isSkipped(phpProject)); assertFalse(sensor.isSkipped(javaProject)); } diff --git a/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/index/DbCloneIndexTest.java b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/index/DbCloneIndexTest.java new file mode 100644 index 00000000000..86698e89d64 --- /dev/null +++ b/plugins/sonar-cpd-plugin/src/test/java/org/sonar/plugins/cpd/index/DbCloneIndexTest.java @@ -0,0 +1,71 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2011 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.plugins.cpd.index; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import java.util.Collection; +import java.util.Iterator; + +import org.junit.Before; +import org.junit.Test; +import org.sonar.duplications.block.Block; +import org.sonar.duplications.block.ByteArray; +import org.sonar.jpa.test.AbstractDbUnitTestCase; + +public class DbCloneIndexTest extends AbstractDbUnitTestCase { + + private DbCloneIndex index; + + @Before + public void setUp() { + index = new DbCloneIndex(getSession(), 5, 4); + } + + @Test(expected = UnsupportedOperationException.class) + public void shouldNotGetByResource() { + index.getByResourceId("foo"); + } + + @Test + public void shouldGetByHash() { + setupData("fixture"); + + index.prepareCache("foo"); + Collection blocks = index.getBySequenceHash(new ByteArray("aa")); + Iterator blocksIterator = blocks.iterator(); + + assertThat(blocks.size(), is(1)); + + Block block = blocksIterator.next(); + assertThat(block.getResourceId(), is("bar-last")); + } + + @Test + public void shouldInsert() { + setupData("fixture"); + + index.insert(new Block("baz", new ByteArray("bb"), 0, 0, 1)); + + checkTables("shouldInsert", "clone_blocks"); + } + +} diff --git a/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/fixture.xml b/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/fixture.xml new file mode 100644 index 00000000000..95599894c1b --- /dev/null +++ b/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/fixture.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/shouldInsert-result.xml b/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/shouldInsert-result.xml new file mode 100644 index 00000000000..ae2767dfedb --- /dev/null +++ b/plugins/sonar-cpd-plugin/src/test/resources/org/sonar/plugins/cpd/index/DbCloneIndexTest/shouldInsert-result.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + -- cgit v1.2.3