From d1e303f8412f61f426a6c746cc2ac10b584940bd Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Mon, 21 Dec 2015 13:54:14 +0100 Subject: [PATCH] SONAR-2867 Standard copy-paste detection should happen within a project, not only within a module --- .../module1/sonar-project.properties | 5 + .../module1/src/main/xoo/sample/File1.xoo | 35 +++ .../src/main/xoo/sample/File1.xoo.measures | 1 + .../module1/src/main/xoo/sample/File2.xoo | 23 ++ .../src/main/xoo/sample/File2.xoo.measures | 1 + .../module2/sonar-project.properties | 5 + .../module2/src/main/xoo/sample/File1.xoo | 35 +++ .../src/main/xoo/sample/File1.xoo.measures | 1 + .../cross-module/sonar-project.properties | 4 + .../CrossModuleDuplicationsTest.java | 152 +++++++++++ .../org/sonar/batch/cpd/CpdComponents.java | 6 +- ...bstractCpdEngine.java => CpdExecutor.java} | 87 ++++++- .../cpd/{CpdEngine.java => CpdIndexer.java} | 5 +- .../java/org/sonar/batch/cpd/CpdSensor.java | 12 +- ...tCpdEngine.java => DefaultCpdIndexer.java} | 78 +----- ...JavaCpdEngine.java => JavaCpdIndexer.java} | 85 +------ .../cpd/index/SonarDuplicationsIndex.java | 10 +- .../org/sonar/batch/phases/PhaseExecutor.java | 15 +- .../batch/scan/ProjectScanContainer.java | 6 + .../batch/cpd/AbstractCpdEngineTest.java | 227 ----------------- .../org/sonar/batch/cpd/CpdExecutorTest.java | 240 ++++++++++++++++++ .../org/sonar/batch/cpd/CpdSensorTest.java | 8 +- ...neTest.java => DefaultCpdIndexerTest.java} | 28 +- .../sonar/batch/cpd/JavaCpdEngineTest.java | 91 ------- .../sonar/batch/cpd/JavaCpdIndexerTest.java | 106 ++++++++ .../batch/mediumtest/cpd/CpdMediumTest.java | 72 +++++- .../sonar/duplications/index/CloneIndex.java | 7 + .../duplications/index/MemoryCloneIndex.java | 7 + .../index/PackedMemoryCloneIndex.java | 133 +++++++--- .../index/PackedMemoryCloneIndexTest.java | 32 +++ 30 files changed, 959 insertions(+), 558 deletions(-) create mode 100644 it/it-projects/duplications/cross-module/module1/sonar-project.properties create mode 100644 it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File1.xoo create mode 100644 it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File1.xoo.measures create mode 100644 it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File2.xoo create mode 100644 it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File2.xoo.measures create mode 100644 it/it-projects/duplications/cross-module/module2/sonar-project.properties create mode 100644 it/it-projects/duplications/cross-module/module2/src/main/xoo/sample/File1.xoo create mode 100644 it/it-projects/duplications/cross-module/module2/src/main/xoo/sample/File1.xoo.measures create mode 100644 it/it-projects/duplications/cross-module/sonar-project.properties create mode 100644 it/it-tests/src/test/java/it/duplication/CrossModuleDuplicationsTest.java rename sonar-batch/src/main/java/org/sonar/batch/cpd/{AbstractCpdEngine.java => CpdExecutor.java} (55%) rename sonar-batch/src/main/java/org/sonar/batch/cpd/{CpdEngine.java => CpdIndexer.java} (90%) rename sonar-batch/src/main/java/org/sonar/batch/cpd/{DefaultCpdEngine.java => DefaultCpdIndexer.java} (54%) rename sonar-batch/src/main/java/org/sonar/batch/cpd/{JavaCpdEngine.java => JavaCpdIndexer.java} (53%) delete mode 100644 sonar-batch/src/test/java/org/sonar/batch/cpd/AbstractCpdEngineTest.java create mode 100644 sonar-batch/src/test/java/org/sonar/batch/cpd/CpdExecutorTest.java rename sonar-batch/src/test/java/org/sonar/batch/cpd/{DefaultCpdEngineTest.java => DefaultCpdIndexerTest.java} (70%) delete mode 100644 sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java create mode 100644 sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdIndexerTest.java diff --git a/it/it-projects/duplications/cross-module/module1/sonar-project.properties b/it/it-projects/duplications/cross-module/module1/sonar-project.properties new file mode 100644 index 00000000000..a4b8c4853a5 --- /dev/null +++ b/it/it-projects/duplications/cross-module/module1/sonar-project.properties @@ -0,0 +1,5 @@ +sonar.projectKey=module1 +sonar.projectName=Module 1 +sonar.projectVersion=1.0-SNAPSHOT +sonar.sources=src/main/xoo +sonar.language=xoo diff --git a/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File1.xoo b/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File1.xoo new file mode 100644 index 00000000000..5e494b196ab --- /dev/null +++ b/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File1.xoo @@ -0,0 +1,35 @@ +package sample; + +public class File1 { + + public File1() { + } + + public void test() { + char[] charList = new char[30]; + for (int i = 0; i < 10; i++) { + charList[i] = 'a'; + } + for (int i = 0; i < 10; i++) { + charList[i] = 'a'; + } + int intergerToBeIncremented = 0; + while (intergerToBeIncremented < 100) { + intergerToBeIncremented++; + } + int intergerToBeIncremented2 = 0; + while (intergerToBeIncremented2 < 100) { + intergerToBeIncremented2++; + } + String temp = ""; + for (int i=0; i<10; i++){ + temp += "say something"+i; + } + for (int i=0; i<20; i++){ + temp += "say nothing"+i; + } + for (int i=0; i<30; i++){ + temp += "always say nothing"+i; + } + } +} diff --git a/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File1.xoo.measures b/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File1.xoo.measures new file mode 100644 index 00000000000..5a79b0b5dbb --- /dev/null +++ b/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File1.xoo.measures @@ -0,0 +1 @@ +ncloc:36 diff --git a/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File2.xoo b/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File2.xoo new file mode 100644 index 00000000000..00b502d423a --- /dev/null +++ b/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File2.xoo @@ -0,0 +1,23 @@ +package sample; + +public class File1 { + + public File1() { + } + + public void otherMethod() { + String temp = ""; + for (int i=0; i<10; i++){ + temp += "say something"+i; + int nothing = 0; + } + for (int i=0; i<20; i++){ + temp += "say nothing"+i; + int nothing = 1; + } + for (int i=0; i<30; i++){ + temp += "always say nothing"+i; + int nothing = 2; + } + } +} diff --git a/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File2.xoo.measures b/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File2.xoo.measures new file mode 100644 index 00000000000..d90983a3e9f --- /dev/null +++ b/it/it-projects/duplications/cross-module/module1/src/main/xoo/sample/File2.xoo.measures @@ -0,0 +1 @@ +ncloc:24 diff --git a/it/it-projects/duplications/cross-module/module2/sonar-project.properties b/it/it-projects/duplications/cross-module/module2/sonar-project.properties new file mode 100644 index 00000000000..0b71b3d7548 --- /dev/null +++ b/it/it-projects/duplications/cross-module/module2/sonar-project.properties @@ -0,0 +1,5 @@ +sonar.projectKey=module2 +sonar.projectName=Module 2 +sonar.projectVersion=1.0-SNAPSHOT +sonar.sources=src/main/xoo +sonar.language=xoo diff --git a/it/it-projects/duplications/cross-module/module2/src/main/xoo/sample/File1.xoo b/it/it-projects/duplications/cross-module/module2/src/main/xoo/sample/File1.xoo new file mode 100644 index 00000000000..cc0b6612812 --- /dev/null +++ b/it/it-projects/duplications/cross-module/module2/src/main/xoo/sample/File1.xoo @@ -0,0 +1,35 @@ +package sample; + +public class File1 { + + public File1() { + } + + public void test2() { + char[] charList = new char[30]; + for (int i = 0; i < 10; i++) { + charList[i] = 'a'; + } + for (int i = 0; i < 10; i++) { + charList[i] = 'a'; + } + int intergerToBeIncremented = 0; + while (intergerToBeIncremented < 100) { + intergerToBeIncremented++; + } + int intergerToBeIncremented2 = 0; + while (intergerToBeIncremented2 < 100) { + intergerToBeIncremented2++; + } + String temp = ""; + for (int i=0; i<10; i++){ + temp += "say something"+i; + } + for (int i=0; i<20; i++){ + temp += "say nothing"+i; + } + for (int i=0; i<30; i++){ + temp += "always say nothing"+i; + } + } +} diff --git a/it/it-projects/duplications/cross-module/module2/src/main/xoo/sample/File1.xoo.measures b/it/it-projects/duplications/cross-module/module2/src/main/xoo/sample/File1.xoo.measures new file mode 100644 index 00000000000..5a79b0b5dbb --- /dev/null +++ b/it/it-projects/duplications/cross-module/module2/src/main/xoo/sample/File1.xoo.measures @@ -0,0 +1 @@ +ncloc:36 diff --git a/it/it-projects/duplications/cross-module/sonar-project.properties b/it/it-projects/duplications/cross-module/sonar-project.properties new file mode 100644 index 00000000000..f4c7496f179 --- /dev/null +++ b/it/it-projects/duplications/cross-module/sonar-project.properties @@ -0,0 +1,4 @@ +sonar.projectKey=cross-module +sonar.projectName=Cross Module Duplication +sonar.projectVersion=1.0-SNAPSHOT +sonar.modules=module1,module2 diff --git a/it/it-tests/src/test/java/it/duplication/CrossModuleDuplicationsTest.java b/it/it-tests/src/test/java/it/duplication/CrossModuleDuplicationsTest.java new file mode 100644 index 00000000000..39bad0abe2e --- /dev/null +++ b/it/it-tests/src/test/java/it/duplication/CrossModuleDuplicationsTest.java @@ -0,0 +1,152 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package it.duplication; + +import com.google.common.collect.ImmutableMap; +import com.sonar.orchestrator.Orchestrator; +import com.sonar.orchestrator.build.SonarRunner; +import com.sonar.orchestrator.locator.FileLocation; +import it.Category4Suite; +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.wsclient.services.Resource; +import org.sonar.wsclient.services.ResourceQuery; +import util.ItUtils; + +import java.io.File; +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CrossModuleDuplicationsTest { + private static final String PROJECT_KEY = "cross-module"; + private static final String PROJECT_DIR = "duplications/" + PROJECT_KEY; + private File projectDir; + + @ClassRule + public static Orchestrator orchestrator = Category4Suite.ORCHESTRATOR; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @BeforeClass + public static void analyzeProjects() { + + } + + @Before + public void setUpProject() throws IOException { + orchestrator.resetData(); + orchestrator.getServer().restoreProfile(FileLocation.ofClasspath("/duplication/xoo-duplication-profile.xml")); + + FileUtils.copyDirectory(ItUtils.projectDir(PROJECT_DIR), temp.getRoot()); + projectDir = temp.getRoot(); + } + + @Test + public void testDuplications() throws IOException { + analyzeProject(projectDir, PROJECT_KEY, true); + verifyDuplicationMeasures(PROJECT_KEY, 2, 54, 2, 56.3); + + // File1 is the one duplicated in both modules + verifyDuplicationMeasures(PROJECT_KEY + ":module1:src/main/xoo/sample/File1.xoo", 1, 27, 1, 75); + verifyDuplicationMeasures(PROJECT_KEY + ":module2:src/main/xoo/sample/File1.xoo", 1, 27, 1, 75); + } + + @Test + // SONAR-6184 + public void testGhostDuplication() throws IOException { + analyzeProject(projectDir, PROJECT_KEY, true); + + verifyDuplicationMeasures(PROJECT_KEY + ":module1", 1, 27, 1, 45); + verifyDuplicationMeasures(PROJECT_KEY + ":module2", 1, 27, 1, 75); + + // move File2 from module1 to module2 + File src = FileUtils.getFile(projectDir, "module1", "src", "main", "xoo", "sample", "File2.xoo"); + File dst = FileUtils.getFile(projectDir, "module2", "src", "main", "xoo", "sample", "File2.xoo"); + FileUtils.moveFile(src, dst); + + src = new File(src.getParentFile(), "File2.xoo.measures"); + dst = new File(dst.getParentFile(), "File2.xoo.measures"); + FileUtils.moveFile(src, dst); + + // duplication should remain unchanged (except for % of duplication) + analyzeProject(projectDir, PROJECT_KEY, false); + verifyDuplicationMeasures(PROJECT_KEY + ":module1", 1, 27, 1, 75); + verifyDuplicationMeasures(PROJECT_KEY + ":module2", 1, 27, 1, 45); + } + + @Test + // SONAR-6184 + public void testDuplicationFix() throws IOException { + analyzeProject(projectDir, PROJECT_KEY, true); + + verifyDuplicationMeasures(PROJECT_KEY + ":module1", 1, 27, 1, 45); + verifyDuplicationMeasures(PROJECT_KEY + ":module2", 1, 27, 1, 75); + + // remove File1 from module1 + File f1 = FileUtils.getFile(projectDir, "module1", "src", "main", "xoo", "sample", "File1.xoo"); + File f1m = FileUtils.getFile(projectDir, "module2", "src", "main", "xoo", "sample", "File1.xoo.measures"); + f1.delete(); + f1m.delete(); + + // duplication should be 0 + analyzeProject(projectDir, PROJECT_KEY, false); + verifyDuplicationMeasures(PROJECT_KEY + ":module1", 0, 0, 0, 0); + verifyDuplicationMeasures(PROJECT_KEY + ":module2", 0, 0, 0, 0); + } + + private static SonarRunner analyzeProject(File projectDir, String projectKey, boolean create, String... additionalProperties) { + if (create) { + orchestrator.getServer().provisionProject(projectKey, projectKey); + orchestrator.getServer().associateProjectToQualityProfile(projectKey, "xoo", "xoo-duplication-profile"); + } + + SonarRunner sonarRunner = SonarRunner.create(projectDir); + ImmutableMap.Builder builder = ImmutableMap.builder(); + + for (int i = 0; i < additionalProperties.length; i += 2) { + builder.put(additionalProperties[i], additionalProperties[i + 1]); + } + SonarRunner scan = sonarRunner.setDebugLogs(true).setProperties(builder.build()); + orchestrator.executeBuild(scan); + return scan; + } + + private static void verifyDuplicationMeasures(String componentKey, int duplicatedBlocks, int duplicatedLines, int duplicatedFiles, double duplicatedLinesDensity) { + Resource file = getComponent(componentKey); + assertThat(file.getMeasureValue("duplicated_blocks").intValue()).isEqualTo(duplicatedBlocks); + assertThat(file.getMeasureValue("duplicated_lines").intValue()).isEqualTo(duplicatedLines); + assertThat(file.getMeasureValue("duplicated_files").intValue()).isEqualTo(duplicatedFiles); + assertThat(file.getMeasureValue("duplicated_lines_density")).isEqualTo(duplicatedLinesDensity); + } + + private static Resource getComponent(String key) { + Resource component = orchestrator.getServer().getWsClient() + .find(ResourceQuery.createForMetrics(key, "duplicated_lines", "duplicated_blocks", "duplicated_files", "duplicated_lines_density")); + assertThat(component).isNotNull(); + return component; + } +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdComponents.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdComponents.java index ffac2cba012..b920ef1cefa 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdComponents.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdComponents.java @@ -27,12 +27,12 @@ public final class CpdComponents { private CpdComponents() { } - public static List all() { + public static List> all() { return ImmutableList.of( CpdSensor.class, CpdMappings.class, - JavaCpdEngine.class, - DefaultCpdEngine.class); + JavaCpdIndexer.class, + DefaultCpdIndexer.class); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/AbstractCpdEngine.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdExecutor.java similarity index 55% rename from sonar-batch/src/main/java/org/sonar/batch/cpd/AbstractCpdEngine.java rename to sonar-batch/src/main/java/org/sonar/batch/cpd/CpdExecutor.java index 8905bf5ac07..302035c3973 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cpd/AbstractCpdEngine.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdExecutor.java @@ -17,45 +17,104 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - package org.sonar.batch.cpd; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; -import java.util.List; +import com.google.common.base.Predicate; import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.config.Settings; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; +import org.sonar.batch.cpd.index.SonarDuplicationsIndex; import org.sonar.batch.index.BatchComponent; import org.sonar.batch.index.BatchComponentCache; import org.sonar.batch.protocol.output.BatchReport; import org.sonar.batch.protocol.output.BatchReport.Duplicate; import org.sonar.batch.protocol.output.BatchReport.Duplication; import org.sonar.batch.report.ReportPublisher; +import org.sonar.duplications.block.Block; +import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm; import org.sonar.duplications.index.CloneGroup; import org.sonar.duplications.index.ClonePart; +import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks; -import static com.google.common.collect.FluentIterable.from; - -public abstract class AbstractCpdEngine extends CpdEngine { +import java.util.Collection; +import java.util.Iterator; +import java.util.List; - private static final Logger LOG = Loggers.get(AbstractCpdEngine.class); +import static com.google.common.collect.FluentIterable.from; +/** + * Runs on the root module, at the end of the project analysis. + * It executes copy paste detection involving all files of all modules, which were indexed during sensors execution for each module + * by {@link CpdSensor). The sensor is responsible for handling exclusions and block sizes. + */ +public class CpdExecutor { + private static final Logger LOG = Loggers.get(CpdExecutor.class); static final int MAX_CLONE_GROUP_PER_FILE = 100; static final int MAX_CLONE_PART_PER_GROUP = 100; + private final SonarDuplicationsIndex index; private final ReportPublisher publisher; private final BatchComponentCache batchComponentCache; + private final Settings settings; - public AbstractCpdEngine(ReportPublisher publisher, BatchComponentCache batchComponentCache) { + public CpdExecutor(Settings settings, SonarDuplicationsIndex index, ReportPublisher publisher, BatchComponentCache batchComponentCache) { + this.settings = settings; + this.index = index; this.publisher = publisher; this.batchComponentCache = batchComponentCache; } - protected final void saveDuplications(final InputFile inputFile, List duplications) { + public void execute() { + Iterator it = index.iterator(); + + while (it.hasNext()) { + ResourceBlocks resourceBlocks = it.next(); + runCpdAnalysis(resourceBlocks.resourceId(), resourceBlocks.blocks()); + } + } + + private void runCpdAnalysis(String resource, Collection fileBlocks) { + LOG.debug("Detection of duplications for {}", resource); + + BatchComponent component = batchComponentCache.get(resource); + if (component == null) { + LOG.error("Resource not found in component cache: {}. Skipping CPD computation for it", resource); + return; + } + + List duplications; + try { + duplications = SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks); + } catch (Exception e) { + throw new IllegalStateException("Fail during detection of duplication for " + resource, e); + } + + InputFile inputFile = (InputFile) component.inputComponent(); + Predicate minimumTokensPredicate = DuplicationPredicates.numberOfUnitsNotLessThan(getMinimumTokens(inputFile.language())); + List filtered = from(duplications).filter(minimumTokensPredicate).toList(); + + saveDuplications(component, filtered); + } + + @VisibleForTesting + int getMinimumTokens(String languageKey) { + int minimumTokens = settings.getInt("sonar.cpd." + languageKey + ".minimumTokens"); + if (minimumTokens == 0) { + minimumTokens = 100; + } + + return minimumTokens; + } + + @VisibleForTesting + final void saveDuplications(final BatchComponent component, List duplications) { if (duplications.size() > MAX_CLONE_GROUP_PER_FILE) { - LOG.warn("Too many duplication groups on file " + inputFile.relativePath() + ". Keep only the first " + MAX_CLONE_GROUP_PER_FILE + " groups."); + LOG.warn("Too many duplication groups on file " + component.inputComponent() + ". Keep only the first " + MAX_CLONE_GROUP_PER_FILE + + " groups."); } - final BatchComponent component = batchComponentCache.get(inputFile); Iterable reportDuplications = from(duplications) .limit(MAX_CLONE_GROUP_PER_FILE) .transform( @@ -65,14 +124,14 @@ public abstract class AbstractCpdEngine extends CpdEngine { @Override public BatchReport.Duplication apply(CloneGroup input) { - return toReportDuplication(component, inputFile, dupBuilder, blockBuilder, input); + return toReportDuplication(component, dupBuilder, blockBuilder, input); } }); publisher.getWriter().writeComponentDuplications(component.batchId(), reportDuplications); } - private Duplication toReportDuplication(BatchComponent component, InputFile inputFile, Duplication.Builder dupBuilder, Duplicate.Builder blockBuilder, CloneGroup input) { + private Duplication toReportDuplication(BatchComponent component, Duplication.Builder dupBuilder, Duplicate.Builder blockBuilder, CloneGroup input) { dupBuilder.clear(); ClonePart originBlock = input.getOriginPart(); blockBuilder.clear(); @@ -85,7 +144,8 @@ public abstract class AbstractCpdEngine extends CpdEngine { if (!duplicate.equals(originBlock)) { clonePartCount++; if (clonePartCount > MAX_CLONE_PART_PER_GROUP) { - LOG.warn("Too many duplication references on file " + inputFile.relativePath() + " for block at line " + originBlock.getStartLine() + ". Keep only the first " + LOG.warn("Too many duplication references on file " + component.inputComponent() + " for block at line " + + originBlock.getStartLine() + ". Keep only the first " + MAX_CLONE_PART_PER_GROUP + " references."); break; } @@ -105,5 +165,4 @@ public abstract class AbstractCpdEngine extends CpdEngine { } return dupBuilder.build(); } - } diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdEngine.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdIndexer.java similarity index 90% rename from sonar-batch/src/main/java/org/sonar/batch/cpd/CpdEngine.java rename to sonar-batch/src/main/java/org/sonar/batch/cpd/CpdIndexer.java index 3b5cfa3b738..6d53f2a4946 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdEngine.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdIndexer.java @@ -21,14 +21,13 @@ package org.sonar.batch.cpd; import org.slf4j.Logger; import org.sonar.api.batch.BatchSide; -import org.sonar.api.batch.sensor.SensorContext; @BatchSide -public abstract class CpdEngine { +public abstract class CpdIndexer { abstract boolean isLanguageSupported(String language); - abstract void analyse(String language, SensorContext context); + abstract void index(String language); protected void logExclusions(String[] exclusions, Logger logger) { if (exclusions.length > 0) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdSensor.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdSensor.java index ed41921c9eb..63ee56ac819 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdSensor.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/CpdSensor.java @@ -35,12 +35,12 @@ public class CpdSensor implements Sensor { private static final Logger LOG = LoggerFactory.getLogger(CpdSensor.class); - private CpdEngine sonarEngine; - private CpdEngine sonarBridgeEngine; + private CpdIndexer sonarEngine; + private CpdIndexer sonarBridgeEngine; private Settings settings; private FileSystem fs; - public CpdSensor(JavaCpdEngine sonarEngine, DefaultCpdEngine sonarBridgeEngine, Settings settings, FileSystem fs) { + public CpdSensor(JavaCpdIndexer sonarEngine, DefaultCpdIndexer sonarBridgeEngine, Settings settings, FileSystem fs) { this.sonarEngine = sonarEngine; this.sonarBridgeEngine = sonarBridgeEngine; this.settings = settings; @@ -54,7 +54,7 @@ public class CpdSensor implements Sensor { } @VisibleForTesting - CpdEngine getEngine(String language) { + CpdIndexer getEngine(String language) { if (sonarEngine.isLanguageSupported(language)) { return sonarEngine; } @@ -87,13 +87,13 @@ public class CpdSensor implements Sensor { continue; } - CpdEngine engine = getEngine(language); + CpdIndexer engine = getEngine(language); if (!engine.isLanguageSupported(language)) { LOG.debug("Detection of duplicated code is not supported for {}", language); continue; } LOG.info("{} is used for {}", engine, language); - engine.analyse(language, context); + engine.index(language); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdIndexer.java similarity index 54% rename from sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java rename to sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdIndexer.java index 2eb64fcc73d..bbc07472f95 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdIndexer.java @@ -20,16 +20,8 @@ package org.sonar.batch.cpd; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Predicate; import com.google.common.collect.Lists; -import java.util.Collection; -import java.util.Collections; import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.CoreProperties; @@ -38,39 +30,25 @@ import org.sonar.api.batch.fs.FilePredicates; import org.sonar.api.batch.fs.FileSystem; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputFile; -import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.config.Settings; import org.sonar.batch.cpd.index.SonarDuplicationsIndex; -import org.sonar.batch.index.BatchComponentCache; -import org.sonar.batch.report.ReportPublisher; import org.sonar.duplications.block.Block; -import org.sonar.duplications.index.CloneGroup; import org.sonar.duplications.internal.pmd.TokenizerBridge; -import static com.google.common.collect.FluentIterable.from; +public class DefaultCpdIndexer extends CpdIndexer { -public class DefaultCpdEngine extends AbstractCpdEngine { - - private static final Logger LOG = LoggerFactory.getLogger(DefaultCpdEngine.class); - - /** - * Limit of time to analyse one file (in seconds). - */ - private static final int TIMEOUT = 5 * 60; + private static final Logger LOG = LoggerFactory.getLogger(DefaultCpdIndexer.class); private final CpdMappings mappings; private final FileSystem fs; private final Settings settings; - private final ReportPublisher publisher; - private final BatchComponentCache batchComponentCache; + private final SonarDuplicationsIndex index; - public DefaultCpdEngine(CpdMappings mappings, FileSystem fs, Settings settings, ReportPublisher publisher, BatchComponentCache batchComponentCache) { - super(publisher, batchComponentCache); + public DefaultCpdIndexer(CpdMappings mappings, FileSystem fs, Settings settings, SonarDuplicationsIndex index) { this.mappings = mappings; this.fs = fs; this.settings = settings; - this.publisher = publisher; - this.batchComponentCache = batchComponentCache; + this.index = index; } @Override @@ -79,7 +57,7 @@ public class DefaultCpdEngine extends AbstractCpdEngine { } @Override - public void analyse(String languageKey, SensorContext context) { + public void index(String languageKey) { CpdMapping mapping = mappings.getMapping(languageKey); if (mapping == null) { LOG.debug("No CpdMapping for language " + languageKey); @@ -98,41 +76,7 @@ public class DefaultCpdEngine extends AbstractCpdEngine { } // Create index - SonarDuplicationsIndex index = new SonarDuplicationsIndex(publisher, batchComponentCache, settings); populateIndex(languageKey, sourceFiles, mapping, index); - - // Detect - runCpdAnalysis(languageKey, context, sourceFiles, index); - } - - private void runCpdAnalysis(String languageKey, SensorContext context, List sourceFiles, SonarDuplicationsIndex index) { - Predicate minimumTokensPredicate = DuplicationPredicates.numberOfUnitsNotLessThan(getMinimumTokens(languageKey)); - - ExecutorService executorService = Executors.newSingleThreadExecutor(); - try { - for (InputFile inputFile : sourceFiles) { - LOG.debug("Detection of duplications for {}", inputFile); - String resourceEffectiveKey = ((DefaultInputFile) inputFile).key(); - Collection fileBlocks = index.getByInputFile(inputFile, resourceEffectiveKey); - - List filtered; - try { - List duplications = executorService.submit(new JavaCpdEngine.Task(index, fileBlocks)).get(TIMEOUT, TimeUnit.SECONDS); - filtered = from(duplications) - .filter(minimumTokensPredicate) - .toList(); - } catch (TimeoutException e) { - filtered = Collections.emptyList(); - LOG.warn("Timeout during detection of duplications for " + inputFile, e); - } catch (InterruptedException | ExecutionException e) { - throw new IllegalStateException("Fail during detection of duplication for " + inputFile, e); - } - - saveDuplications(inputFile, filtered); - } - } finally { - executorService.shutdown(); - } } private void populateIndex(String languageKey, List sourceFiles, CpdMapping mapping, SonarDuplicationsIndex index) { @@ -165,14 +109,4 @@ public class DefaultCpdEngine extends AbstractCpdEngine { } } - @VisibleForTesting - int getMinimumTokens(String languageKey) { - int minimumTokens = settings.getInt("sonar.cpd." + languageKey + ".minimumTokens"); - if (minimumTokens == 0) { - minimumTokens = 100; - } - - return minimumTokens; - } - } diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdIndexer.java similarity index 53% rename from sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java rename to sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdIndexer.java index d83e049a45a..42fdf42e000 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdIndexer.java @@ -25,15 +25,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.InputStreamReader; import java.io.Reader; -import java.util.Collection; -import java.util.Collections; import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,44 +34,30 @@ import org.sonar.api.batch.fs.FilePredicates; import org.sonar.api.batch.fs.FileSystem; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputFile; -import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.config.Settings; import org.sonar.batch.cpd.index.SonarDuplicationsIndex; -import org.sonar.batch.index.BatchComponentCache; -import org.sonar.batch.report.ReportPublisher; import org.sonar.duplications.block.Block; import org.sonar.duplications.block.BlockChunker; -import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm; -import org.sonar.duplications.index.CloneGroup; -import org.sonar.duplications.index.CloneIndex; import org.sonar.duplications.java.JavaStatementBuilder; import org.sonar.duplications.java.JavaTokenProducer; import org.sonar.duplications.statement.Statement; import org.sonar.duplications.statement.StatementChunker; import org.sonar.duplications.token.TokenChunker; -public class JavaCpdEngine extends AbstractCpdEngine { +public class JavaCpdIndexer extends CpdIndexer { - private static final Logger LOG = LoggerFactory.getLogger(JavaCpdEngine.class); + private static final Logger LOG = LoggerFactory.getLogger(JavaCpdIndexer.class); private static final int BLOCK_SIZE = 10; - /** - * Limit of time to analyse one file (in seconds). - */ - private static final int TIMEOUT = 5 * 60; - private final FileSystem fs; private final Settings settings; - private final ReportPublisher publisher; - private final BatchComponentCache batchComponentCache; + private final SonarDuplicationsIndex index; - public JavaCpdEngine(FileSystem fs, Settings settings, ReportPublisher publisher, BatchComponentCache batchComponentCache) { - super(publisher, batchComponentCache); + public JavaCpdIndexer(FileSystem fs, Settings settings, SonarDuplicationsIndex index) { this.fs = fs; this.settings = settings; - this.publisher = publisher; - this.batchComponentCache = batchComponentCache; + this.index = index; } @Override @@ -88,7 +66,7 @@ public class JavaCpdEngine extends AbstractCpdEngine { } @Override - public void analyse(String languageKey, SensorContext context) { + public void index(String languageKey) { String[] cpdExclusions = settings.getStringArray(CoreProperties.CPD_EXCLUSIONS); logExclusions(cpdExclusions, LOG); FilePredicates p = fs.predicates(); @@ -99,13 +77,10 @@ public class JavaCpdEngine extends AbstractCpdEngine { if (sourceFiles.isEmpty()) { return; } - SonarDuplicationsIndex index = createIndex(sourceFiles); - detect(index, context, sourceFiles); + createIndex(sourceFiles); } - private SonarDuplicationsIndex createIndex(Iterable sourceFiles) { - final SonarDuplicationsIndex index = new SonarDuplicationsIndex(publisher, batchComponentCache, settings); - + private void createIndex(Iterable sourceFiles) { TokenChunker tokenChunker = JavaTokenProducer.build(); StatementChunker statementChunker = JavaStatementBuilder.build(); BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE); @@ -129,49 +104,5 @@ public class JavaCpdEngine extends AbstractCpdEngine { List blocks = blockChunker.chunk(resourceEffectiveKey, statements); index.insert(inputFile, blocks); } - - return index; } - - private void detect(SonarDuplicationsIndex index, org.sonar.api.batch.sensor.SensorContext context, List sourceFiles) { - ExecutorService executorService = Executors.newSingleThreadExecutor(); - try { - for (InputFile inputFile : sourceFiles) { - LOG.debug("Detection of duplications for {}", inputFile); - String resourceEffectiveKey = ((DefaultInputFile) inputFile).key(); - - Collection fileBlocks = index.getByInputFile(inputFile, resourceEffectiveKey); - - List clones; - try { - clones = executorService.submit(new Task(index, fileBlocks)).get(TIMEOUT, TimeUnit.SECONDS); - } catch (TimeoutException e) { - clones = Collections.emptyList(); - LOG.warn("Timeout during detection of duplications for " + inputFile, e); - } catch (InterruptedException | ExecutionException e) { - throw new IllegalStateException("Fail during detection of duplication for " + inputFile, e); - } - - saveDuplications(inputFile, clones); - } - } finally { - executorService.shutdown(); - } - } - - static class Task implements Callable> { - private final CloneIndex index; - private final Collection fileBlocks; - - public Task(CloneIndex index, Collection fileBlocks) { - this.index = index; - this.fileBlocks = fileBlocks; - } - - @Override - public List call() { - return SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks); - } - } - } diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarDuplicationsIndex.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarDuplicationsIndex.java index f5872efdd50..1d55b45f008 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarDuplicationsIndex.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/index/SonarDuplicationsIndex.java @@ -22,6 +22,8 @@ package org.sonar.batch.cpd.index; import com.google.common.base.Function; import com.google.common.collect.Iterables; import java.util.Collection; +import java.util.Iterator; + import org.apache.commons.lang.StringUtils; import org.sonar.api.CoreProperties; import org.sonar.api.batch.fs.InputFile; @@ -34,6 +36,7 @@ import org.sonar.duplications.block.ByteArray; import org.sonar.duplications.index.AbstractCloneIndex; import org.sonar.duplications.index.CloneIndex; import org.sonar.duplications.index.PackedMemoryCloneIndex; +import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks; public class SonarDuplicationsIndex extends AbstractCloneIndex { @@ -76,7 +79,7 @@ public class SonarDuplicationsIndex extends AbstractCloneIndex { && StringUtils.isBlank(settings.getString(CoreProperties.PROJECT_BRANCH_PROPERTY)); } - public Collection getByInputFile(InputFile inputFile, String resourceKey) { + public Collection getByInputFile(String resourceKey) { return mem.getByResourceId(resourceKey); } @@ -95,4 +98,9 @@ public class SonarDuplicationsIndex extends AbstractCloneIndex { throw new UnsupportedOperationException(); } + @Override + public Iterator iterator() { + return mem.iterator(); + } + } diff --git a/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java b/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java index 3919f4d15af..8e2b7c36693 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java +++ b/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java @@ -22,6 +22,7 @@ package org.sonar.batch.phases; import org.sonar.api.batch.SensorContext; import org.sonar.api.resources.Project; import org.sonar.batch.analysis.DefaultAnalysisMode; +import org.sonar.batch.cpd.CpdExecutor; import org.sonar.batch.events.BatchStepEvent; import org.sonar.batch.events.EventBus; import org.sonar.batch.index.DefaultIndex; @@ -52,12 +53,14 @@ public final class PhaseExecutor { private final DefaultAnalysisMode analysisMode; private final IssueTransition localIssueTracking; private final IssueCallback issueCallback; + private final CpdExecutor cpdExecutor; public PhaseExecutor(InitializersExecutor initializersExecutor, PostJobsExecutor postJobsExecutor, SensorsExecutor sensorsExecutor, SensorContext sensorContext, DefaultIndex index, EventBus eventBus, ReportPublisher reportPublisher, ProjectInitializer pi, FileSystemLogger fsLogger, IssuesReports jsonReport, DefaultModuleFileSystem fs, QProfileVerifier profileVerifier, - IssueExclusionsLoader issueExclusionsLoader, DefaultAnalysisMode analysisMode, IssueTransition localIssueTracking, IssueCallback issueCallback) { + IssueExclusionsLoader issueExclusionsLoader, DefaultAnalysisMode analysisMode, IssueTransition localIssueTracking, IssueCallback issueCallback, + CpdExecutor cpdExecutor) { this.postJobsExecutor = postJobsExecutor; this.initializersExecutor = initializersExecutor; this.sensorsExecutor = sensorsExecutor; @@ -74,6 +77,7 @@ public final class PhaseExecutor { this.analysisMode = analysisMode; this.localIssueTracking = localIssueTracking; this.issueCallback = issueCallback; + this.cpdExecutor = cpdExecutor; } /** @@ -101,6 +105,8 @@ public final class PhaseExecutor { if (analysisMode.isIssues()) { localIssueTracking(); issuesCallback(); + } else { + computeDuplications(); } issuesReport(); publishReportJob(); @@ -110,6 +116,13 @@ public final class PhaseExecutor { eventBus.fireEvent(new ProjectAnalysisEvent(module, false)); } + private void computeDuplications() { + String stepName = "Computing duplications"; + eventBus.fireEvent(new BatchStepEvent(stepName, true)); + cpdExecutor.execute(); + eventBus.fireEvent(new BatchStepEvent(stepName, false)); + } + private void publishReportJob() { String stepName = "Publish report"; eventBus.fireEvent(new BatchStepEvent(stepName, true)); diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java index 427f5c3fb7d..767f419920d 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java @@ -41,6 +41,8 @@ import org.sonar.batch.bootstrap.ExtensionMatcher; import org.sonar.batch.bootstrap.ExtensionUtils; import org.sonar.batch.bootstrap.MetricProvider; import org.sonar.batch.cache.ProjectPersistentCacheProvider; +import org.sonar.batch.cpd.CpdExecutor; +import org.sonar.batch.cpd.index.SonarDuplicationsIndex; import org.sonar.batch.events.EventBus; import org.sonar.batch.index.BatchComponentCache; import org.sonar.batch.index.Caches; @@ -203,6 +205,10 @@ public class ProjectScanContainer extends ComponentContainer { CoveragePublisher.class, SourcePublisher.class, TestExecutionAndCoveragePublisher.class, + + // Cpd + CpdExecutor.class, + SonarDuplicationsIndex.class, ScanTaskObservers.class, UserRepositoryLoader.class); diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/AbstractCpdEngineTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/AbstractCpdEngineTest.java deleted file mode 100644 index 1f843d8e284..00000000000 --- a/sonar-batch/src/test/java/org/sonar/batch/cpd/AbstractCpdEngineTest.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.batch.cpd; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.sonar.api.batch.fs.internal.DefaultInputFile; -import org.sonar.api.batch.fs.internal.DefaultInputModule; -import org.sonar.api.batch.sensor.SensorContext; -import org.sonar.api.resources.Project; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.batch.index.BatchComponent; -import org.sonar.batch.index.BatchComponentCache; -import org.sonar.batch.protocol.output.BatchReport.Duplication; -import org.sonar.batch.protocol.output.BatchReportReader; -import org.sonar.batch.protocol.output.BatchReportWriter; -import org.sonar.batch.report.ReportPublisher; -import org.sonar.core.util.CloseableIterator; -import org.sonar.duplications.index.CloneGroup; -import org.sonar.duplications.index.ClonePart; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class AbstractCpdEngineTest { - - @Rule - public LogTester logTester = new LogTester(); - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - private AbstractCpdEngine engine; - - private BatchReportReader reader; - private DefaultInputFile inputFile1; - private BatchComponent batchComponent1; - private BatchComponent batchComponent2; - private BatchComponent batchComponent3; - - @Before - public void before() throws IOException { - File outputDir = temp.newFolder(); - ReportPublisher reportPublisher = mock(ReportPublisher.class); - when(reportPublisher.getWriter()).thenReturn(new BatchReportWriter(outputDir)); - reader = new BatchReportReader(outputDir); - BatchComponentCache componentCache = new BatchComponentCache(); - Project p = new Project("foo"); - componentCache.add(p, null).setInputComponent(new DefaultInputModule("foo")); - org.sonar.api.resources.Resource sampleFile = org.sonar.api.resources.File.create("src/Foo.php").setEffectiveKey("foo:src/Foo.php"); - inputFile1 = new DefaultInputFile("foo", "src/Foo.php").setLines(5); - batchComponent1 = componentCache.add(sampleFile, null).setInputComponent(inputFile1); - org.sonar.api.resources.Resource sampleFile2 = org.sonar.api.resources.File.create("src/Foo2.php").setEffectiveKey("foo:src/Foo2.php"); - batchComponent2 = componentCache.add(sampleFile2, null).setInputComponent(new DefaultInputFile("foo", "src/Foo2.php").setLines(5)); - org.sonar.api.resources.Resource sampleFile3 = org.sonar.api.resources.File.create("src/Foo3.php").setEffectiveKey("foo:src/Foo3.php"); - batchComponent3 = componentCache.add(sampleFile3, null).setInputComponent(new DefaultInputFile("foo", "src/Foo3.php").setLines(5)); - engine = new AbstractCpdEngine(reportPublisher, componentCache) { - - @Override - boolean isLanguageSupported(String language) { - return false; - } - - @Override - void analyse(String language, SensorContext context) { - } - }; - } - - @Test - public void testNothingToSave() { - engine.saveDuplications(inputFile1, Collections.EMPTY_LIST); - - assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(0); - } - - @Test - public void testOneSimpleDuplicationBetweenTwoFiles() { - List groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 2, 4), new ClonePart(batchComponent2.key(), 0, 15, 17))); - engine.saveDuplications(inputFile1, groups); - - assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1); - CloseableIterator dups = reader.readComponentDuplications(batchComponent1.batchId()); - Duplication duplication = dups.next(); - dups.close(); - assertThat(duplication.getOriginPosition().getStartLine()).isEqualTo(2); - assertThat(duplication.getOriginPosition().getEndLine()).isEqualTo(4); - assertThat(duplication.getDuplicateList()).hasSize(1); - assertThat(duplication.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent2.batchId()); - assertThat(duplication.getDuplicate(0).getRange().getStartLine()).isEqualTo(15); - assertThat(duplication.getDuplicate(0).getRange().getEndLine()).isEqualTo(17); - } - - @Test - public void testDuplicationOnSameFile() throws Exception { - List groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent1.key(), 0, 215, 414))); - engine.saveDuplications(inputFile1, groups); - - assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1); - CloseableIterator dups = reader.readComponentDuplications(batchComponent1.batchId()); - Duplication duplication = dups.next(); - dups.close(); - assertThat(duplication.getOriginPosition().getStartLine()).isEqualTo(5); - assertThat(duplication.getOriginPosition().getEndLine()).isEqualTo(204); - assertThat(duplication.getDuplicateList()).hasSize(1); - assertThat(duplication.getDuplicate(0).hasOtherFileRef()).isFalse(); - assertThat(duplication.getDuplicate(0).getRange().getStartLine()).isEqualTo(215); - assertThat(duplication.getDuplicate(0).getRange().getEndLine()).isEqualTo(414); - } - - @Test - public void testTooManyDuplicates() throws Exception { - // 1 origin part + 101 duplicates = 102 - List parts = new ArrayList<>(AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP + 2); - for (int i = 0; i < AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP + 2; i++) { - parts.add(new ClonePart(batchComponent1.key(), i, i, i + 1)); - } - List groups = Arrays.asList(CloneGroup.builder().setLength(0).setOrigin(parts.get(0)).setParts(parts).build()); - engine.saveDuplications(inputFile1, groups); - - assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1); - CloseableIterator dups = reader.readComponentDuplications(batchComponent1.batchId()); - Duplication duplication = dups.next(); - dups.close(); - assertThat(duplication.getDuplicateList()).hasSize(AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP); - - assertThat(logTester.logs(LoggerLevel.WARN)).contains("Too many duplication references on file " + inputFile1.relativePath() + " for block at line 0. Keep only the first " - + AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP + " references."); - } - - @Test - public void testTooManyDuplications() throws Exception { - // 1 origin part + 101 duplicates = 102 - List dups = new ArrayList<>(AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE + 1); - for (int i = 0; i < AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE + 1; i++) { - ClonePart clonePart = new ClonePart(batchComponent1.key(), i, i, i + 1); - ClonePart dupPart = new ClonePart(batchComponent1.key(), i + 1, i + 1, i + 2); - dups.add(newCloneGroup(clonePart, dupPart)); - } - engine.saveDuplications(inputFile1, dups); - - assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE); - - assertThat(logTester.logs(LoggerLevel.WARN)) - .contains("Too many duplication groups on file " + inputFile1.relativePath() + ". Keep only the first " + AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE + " groups."); - } - - @Test - public void testOneDuplicatedGroupInvolvingMoreThanTwoFiles() throws Exception { - List groups = Arrays - .asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 25, 224))); - engine.saveDuplications(inputFile1, groups); - - assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1); - CloseableIterator dups = reader.readComponentDuplications(batchComponent1.batchId()); - Duplication duplication = dups.next(); - dups.close(); - assertThat(duplication.getOriginPosition().getStartLine()).isEqualTo(5); - assertThat(duplication.getOriginPosition().getEndLine()).isEqualTo(204); - assertThat(duplication.getDuplicateList()).hasSize(2); - assertThat(duplication.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent2.batchId()); - assertThat(duplication.getDuplicate(0).getRange().getStartLine()).isEqualTo(15); - assertThat(duplication.getDuplicate(0).getRange().getEndLine()).isEqualTo(214); - assertThat(duplication.getDuplicate(1).getOtherFileRef()).isEqualTo(batchComponent3.batchId()); - assertThat(duplication.getDuplicate(1).getRange().getStartLine()).isEqualTo(25); - assertThat(duplication.getDuplicate(1).getRange().getEndLine()).isEqualTo(224); - } - - @Test - public void testTwoDuplicatedGroupsInvolvingThreeFiles() throws Exception { - List groups = Arrays.asList( - newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214)), - newCloneGroup(new ClonePart(batchComponent1.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 15, 214))); - engine.saveDuplications(inputFile1, groups); - - assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(2); - CloseableIterator dups = reader.readComponentDuplications(batchComponent1.batchId()); - Duplication duplication1 = dups.next(); - Duplication duplication2 = dups.next(); - dups.close(); - assertThat(duplication1.getOriginPosition().getStartLine()).isEqualTo(5); - assertThat(duplication1.getOriginPosition().getEndLine()).isEqualTo(204); - assertThat(duplication1.getDuplicateList()).hasSize(1); - assertThat(duplication1.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent2.batchId()); - assertThat(duplication1.getDuplicate(0).getRange().getStartLine()).isEqualTo(15); - assertThat(duplication1.getDuplicate(0).getRange().getEndLine()).isEqualTo(214); - - assertThat(duplication2.getOriginPosition().getStartLine()).isEqualTo(15); - assertThat(duplication2.getOriginPosition().getEndLine()).isEqualTo(214); - assertThat(duplication2.getDuplicateList()).hasSize(1); - assertThat(duplication2.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent3.batchId()); - assertThat(duplication2.getDuplicate(0).getRange().getStartLine()).isEqualTo(15); - assertThat(duplication2.getDuplicate(0).getRange().getEndLine()).isEqualTo(214); - } - - private CloneGroup newCloneGroup(ClonePart... parts) { - return CloneGroup.builder().setLength(0).setOrigin(parts[0]).setParts(Arrays.asList(parts)).build(); - } - -} diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdExecutorTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdExecutorTest.java new file mode 100644 index 00000000000..4490a21e828 --- /dev/null +++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdExecutorTest.java @@ -0,0 +1,240 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.cpd; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.internal.DefaultInputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.config.Settings; +import org.sonar.api.resources.Project; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.batch.cpd.index.SonarDuplicationsIndex; +import org.sonar.batch.index.BatchComponent; +import org.sonar.batch.index.BatchComponentCache; +import org.sonar.batch.protocol.output.BatchReportReader; +import org.sonar.batch.protocol.output.BatchReportWriter; +import org.sonar.batch.protocol.output.BatchReport.Duplicate; +import org.sonar.batch.protocol.output.BatchReport.Duplication; +import org.sonar.batch.report.ReportPublisher; +import org.sonar.core.util.CloseableIterator; +import org.sonar.duplications.index.CloneGroup; +import org.sonar.duplications.index.ClonePart; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class CpdExecutorTest { + private CpdExecutor executor; + private Settings settings; + private SonarDuplicationsIndex index; + private ReportPublisher publisher; + private BatchComponentCache componentCache; + + @Rule + public LogTester logTester = new LogTester(); + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + // private AbstractCpdEngine engine; + + private BatchReportReader reader; + private BatchComponent batchComponent1; + private BatchComponent batchComponent2; + private BatchComponent batchComponent3; + + @Before + public void setUp() throws IOException { + File outputDir = temp.newFolder(); + + settings = new Settings(); + index = mock(SonarDuplicationsIndex.class); + publisher = mock(ReportPublisher.class); + when(publisher.getWriter()).thenReturn(new BatchReportWriter(outputDir)); + componentCache = new BatchComponentCache(); + executor = new CpdExecutor(settings, index, publisher, componentCache); + reader = new BatchReportReader(outputDir); + + Project p = new Project("foo"); + componentCache.add(p, null).setInputComponent(new DefaultInputModule("foo")); + + batchComponent1 = createComponent("src/Foo.php", 5); + batchComponent2 = createComponent("src/Foo2.php", 5); + batchComponent3 = createComponent("src/Foo3.php", 5); + } + + private BatchComponent createComponent(String relativePath, int lines) { + org.sonar.api.resources.Resource sampleFile = org.sonar.api.resources.File.create("relativePath").setEffectiveKey("foo:" + relativePath); + return componentCache.add(sampleFile, null).setInputComponent(new DefaultInputFile("foo", relativePath).setLines(lines)); + } + + @Test + public void defaultMinimumTokens() { + assertThat(executor.getMinimumTokens("java")).isEqualTo(100); + } + + @Test + public void minimumTokensByLanguage() { + settings.setProperty("sonar.cpd.java.minimumTokens", "42"); + settings.setProperty("sonar.cpd.php.minimumTokens", "33"); + assertThat(executor.getMinimumTokens("java")).isEqualTo(42); + + settings.setProperty("sonar.cpd.java.minimumTokens", "42"); + settings.setProperty("sonar.cpd.php.minimumTokens", "33"); + assertThat(executor.getMinimumTokens("php")).isEqualTo(33); + } + + @Test + public void testNothingToSave() { + executor.saveDuplications(batchComponent1, Collections.emptyList()); + assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(0); + } + + @Test + public void reportOneSimpleDuplicationBetweenTwoFiles() { + List groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 2, 4), new ClonePart(batchComponent2.key(), 0, 15, 17))); + + executor.saveDuplications(batchComponent1, groups); + + Duplication[] dups = readDuplications(1); + assertDuplication(dups[0], 2, 4, batchComponent2.batchId(), 15, 17); + } + + @Test + public void reportDuplicationOnSameFile() throws Exception { + List groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent1.key(), 0, 215, 414))); + executor.saveDuplications(batchComponent1, groups); + + Duplication[] dups = readDuplications(1); + assertDuplication(dups[0], 5, 204, null, 215, 414); + } + + @Test + public void reportTooManyDuplicates() throws Exception { + // 1 origin part + 101 duplicates = 102 + List parts = new ArrayList<>(CpdExecutor.MAX_CLONE_PART_PER_GROUP + 2); + for (int i = 0; i < CpdExecutor.MAX_CLONE_PART_PER_GROUP + 2; i++) { + parts.add(new ClonePart(batchComponent1.key(), i, i, i + 1)); + } + List groups = Arrays.asList(CloneGroup.builder().setLength(0).setOrigin(parts.get(0)).setParts(parts).build()); + executor.saveDuplications(batchComponent1, groups); + + Duplication[] dups = readDuplications(1); + assertThat(dups[0].getDuplicateList()).hasSize(CpdExecutor.MAX_CLONE_PART_PER_GROUP); + + assertThat(logTester.logs(LoggerLevel.WARN)) + .contains("Too many duplication references on file " + batchComponent1.inputComponent() + " for block at line 0. Keep only the first " + + CpdExecutor.MAX_CLONE_PART_PER_GROUP + " references."); + } + + @Test + public void reportTooManyDuplications() throws Exception { + // 1 origin part + 101 duplicates = 102 + List dups = new ArrayList<>(CpdExecutor.MAX_CLONE_GROUP_PER_FILE + 1); + for (int i = 0; i < CpdExecutor.MAX_CLONE_GROUP_PER_FILE + 1; i++) { + ClonePart clonePart = new ClonePart(batchComponent1.key(), i, i, i + 1); + ClonePart dupPart = new ClonePart(batchComponent1.key(), i + 1, i + 1, i + 2); + dups.add(newCloneGroup(clonePart, dupPart)); + } + executor.saveDuplications(batchComponent1, dups); + + assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(CpdExecutor.MAX_CLONE_GROUP_PER_FILE); + + assertThat(logTester.logs(LoggerLevel.WARN)) + .contains("Too many duplication groups on file " + batchComponent1.inputComponent() + ". Keep only the first " + CpdExecutor.MAX_CLONE_GROUP_PER_FILE + " groups."); + } + + @Test + public void reportOneDuplicatedGroupInvolvingMoreThanTwoFiles() throws Exception { + List groups = Arrays + .asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 25, 224))); + executor.saveDuplications(batchComponent1, groups); + + Duplication[] dups = readDuplications(1); + assertDuplication(dups[0], 5, 204, 2); + assertDuplicate(dups[0].getDuplicate(0), batchComponent2.batchId(), 15, 214); + assertDuplicate(dups[0].getDuplicate(1), batchComponent3.batchId(), 25, 224); + } + + @Test + public void reportTwoDuplicatedGroupsInvolvingThreeFiles() throws Exception { + List groups = Arrays.asList( + newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214)), + newCloneGroup(new ClonePart(batchComponent1.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 15, 214))); + executor.saveDuplications(batchComponent1, groups); + + Duplication[] dups = readDuplications(2); + assertDuplication(dups[0], 5, 204, batchComponent2.batchId(), 15, 214); + assertDuplication(dups[1], 15, 214, batchComponent3.batchId(), 15, 214); + } + + private Duplication[] readDuplications(int expected) { + assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(expected); + Duplication[] duplications = new Duplication[expected]; + CloseableIterator dups = reader.readComponentDuplications(batchComponent1.batchId()); + + for(int i = 0; i< expected; i++) { + duplications[i] = dups.next(); + } + dups.close(); + return duplications; + } + + private void assertDuplicate(Duplicate d, int otherFileRef, int rangeStartLine, int rangeEndLine) { + assertThat(d.getOtherFileRef()).isEqualTo(otherFileRef); + assertThat(d.getRange().getStartLine()).isEqualTo(rangeStartLine); + assertThat(d.getRange().getEndLine()).isEqualTo(rangeEndLine); + } + + private void assertDuplication(Duplication d, int originStartLine, int originEndLine, int numDuplicates) { + assertThat(d.getOriginPosition().getStartLine()).isEqualTo(originStartLine); + assertThat(d.getOriginPosition().getEndLine()).isEqualTo(originEndLine); + assertThat(d.getDuplicateList()).hasSize(numDuplicates); + } + + private void assertDuplication(Duplication d, int originStartLine, int originEndLine, Integer otherFileRef, int rangeStartLine, int rangeEndLine) { + assertThat(d.getOriginPosition().getStartLine()).isEqualTo(originStartLine); + assertThat(d.getOriginPosition().getEndLine()).isEqualTo(originEndLine); + assertThat(d.getDuplicateList()).hasSize(1); + if(otherFileRef != null) { + assertThat(d.getDuplicate(0).getOtherFileRef()).isEqualTo(otherFileRef); + } else { + assertThat(d.getDuplicate(0).hasOtherFileRef()).isFalse(); + } + assertThat(d.getDuplicate(0).getRange().getStartLine()).isEqualTo(rangeStartLine); + assertThat(d.getDuplicate(0).getRange().getEndLine()).isEqualTo(rangeEndLine); + } + + private CloneGroup newCloneGroup(ClonePart... parts) { + return CloneGroup.builder().setLength(0).setOrigin(parts[0]).setParts(Arrays.asList(parts)).build(); + } +} diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java index 5c5adb3a4c2..948f008918b 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java @@ -36,15 +36,15 @@ public class CpdSensorTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); - JavaCpdEngine sonarEngine; - DefaultCpdEngine sonarBridgeEngine; + JavaCpdIndexer sonarEngine; + DefaultCpdIndexer sonarBridgeEngine; CpdSensor sensor; Settings settings; @Before public void setUp() throws IOException { - sonarEngine = new JavaCpdEngine(null, null, null, null); - sonarBridgeEngine = new DefaultCpdEngine(new CpdMappings(), null, null, null, null); + sonarEngine = new JavaCpdIndexer(null, null, null); + sonarBridgeEngine = new DefaultCpdIndexer(new CpdMappings(), null, null, null); settings = new Settings(new PropertyDefinitions(CpdComponents.class)); DefaultFileSystem fs = new DefaultFileSystem(temp.newFolder().toPath()); diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdIndexerTest.java similarity index 70% rename from sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java rename to sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdIndexerTest.java index 427c62a1e4c..51f082f5f69 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdIndexerTest.java @@ -31,15 +31,15 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -public class DefaultCpdEngineTest { +public class DefaultCpdIndexerTest { - private DefaultCpdEngine engine; + private DefaultCpdIndexer engine; private Settings settings; @Before public void init() { settings = new Settings(); - engine = new DefaultCpdEngine(null, null, settings, null, null); + engine = new DefaultCpdIndexer(null, null, settings, null); } @Test @@ -59,9 +59,9 @@ public class DefaultCpdEngineTest { @Test public void shouldReturnDefaultBlockSize() { - assertThat(DefaultCpdEngine.getDefaultBlockSize("cobol")).isEqualTo(30); - assertThat(DefaultCpdEngine.getDefaultBlockSize("abap")).isEqualTo(20); - assertThat(DefaultCpdEngine.getDefaultBlockSize("other")).isEqualTo(10); + assertThat(DefaultCpdIndexer.getDefaultBlockSize("cobol")).isEqualTo(30); + assertThat(DefaultCpdIndexer.getDefaultBlockSize("abap")).isEqualTo(20); + assertThat(DefaultCpdIndexer.getDefaultBlockSize("other")).isEqualTo(10); } @Test @@ -77,20 +77,4 @@ public class DefaultCpdEngineTest { assertThat(engine.getBlockSize("cobol")).isEqualTo(42); } - @Test - public void defaultMinimumTokens() { - assertThat(engine.getMinimumTokens("java")).isEqualTo(100); - } - - @Test - public void minimumTokensByLanguage() { - settings.setProperty("sonar.cpd.java.minimumTokens", "42"); - settings.setProperty("sonar.cpd.php.minimumTokens", "33"); - assertThat(engine.getMinimumTokens("java")).isEqualTo(42); - - settings.setProperty("sonar.cpd.java.minimumTokens", "42"); - settings.setProperty("sonar.cpd.php.minimumTokens", "33"); - assertThat(engine.getMinimumTokens("php")).isEqualTo(33); - } - } diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java deleted file mode 100644 index 798f821d1ab..00000000000 --- a/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.batch.cpd; - -import java.io.File; -import org.apache.commons.io.FileUtils; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.sonar.api.batch.fs.internal.DefaultFileSystem; -import org.sonar.api.batch.fs.internal.DefaultInputFile; -import org.sonar.api.batch.sensor.SensorContext; -import org.sonar.api.config.Settings; -import org.sonar.batch.index.BatchComponentCache; -import org.sonar.batch.protocol.output.BatchReport; -import org.sonar.batch.protocol.output.BatchReport.Duplication; -import org.sonar.batch.protocol.output.BatchReportReader; -import org.sonar.batch.protocol.output.BatchReportWriter; -import org.sonar.batch.report.ReportPublisher; -import org.sonar.core.util.CloseableIterator; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class JavaCpdEngineTest { - - private static final String JAVA = "java"; - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - @Test - public void testJavaCodeWithTwoCloneGroupAtSameLines() throws Exception { - - File baseDir = temp.newFolder(); - DefaultFileSystem fs = new DefaultFileSystem(baseDir); - DefaultInputFile file = new DefaultInputFile("foo", "src/ManyStatements.java").setLanguage(JAVA); - fs.add(file); - BatchComponentCache batchComponentCache = new BatchComponentCache(); - batchComponentCache.add(org.sonar.api.resources.File.create("src/Foo.java").setEffectiveKey("foo:src/ManyStatements.java"), null).setInputComponent(file); - File ioFile = file.file(); - FileUtils.copyURLToFile(this.getClass().getResource("ManyStatements.java"), ioFile); - - File reportOut = temp.newFolder(); - ReportPublisher reportPublisher = mock(ReportPublisher.class); - when(reportPublisher.getWriter()).thenReturn(new BatchReportWriter(reportOut)); - JavaCpdEngine engine = new JavaCpdEngine(fs, new Settings(), reportPublisher, batchComponentCache); - engine.analyse(JAVA, mock(SensorContext.class)); - - BatchReportReader reader = new BatchReportReader(reportOut); - try (CloseableIterator it = reader.readComponentDuplications(1)) { - Duplication dupGroup1 = it.next(); - Duplication dupGroup2 = it.next(); - - assertThat(dupGroup1.getOriginPosition().getStartLine()).isEqualTo(6); - assertThat(dupGroup1.getOriginPosition().getEndLine()).isEqualTo(6); - assertThat(dupGroup1.getDuplicateCount()).isEqualTo(1); - assertThat(dupGroup1.getDuplicate(0).getRange().getStartLine()).isEqualTo(8); - assertThat(dupGroup1.getDuplicate(0).getRange().getEndLine()).isEqualTo(8); - - assertThat(dupGroup2.getOriginPosition().getStartLine()).isEqualTo(6); - assertThat(dupGroup2.getOriginPosition().getEndLine()).isEqualTo(6); - assertThat(dupGroup2.getDuplicateCount()).isEqualTo(2); - assertThat(dupGroup2.getDuplicate(0).getRange().getStartLine()).isEqualTo(7); - assertThat(dupGroup2.getDuplicate(0).getRange().getEndLine()).isEqualTo(7); - assertThat(dupGroup2.getDuplicate(1).getRange().getStartLine()).isEqualTo(8); - assertThat(dupGroup2.getDuplicate(1).getRange().getEndLine()).isEqualTo(8); - } catch (Exception e) { - throw new IllegalStateException(e); - } - - } - -} diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdIndexerTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdIndexerTest.java new file mode 100644 index 00000000000..4b5ff488828 --- /dev/null +++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdIndexerTest.java @@ -0,0 +1,106 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.cpd; + +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.sonar.api.CoreProperties; +import org.sonar.api.batch.fs.FileSystem; +import org.sonar.api.batch.fs.internal.DefaultFileSystem; +import org.sonar.api.batch.fs.internal.DefaultInputFile; +import org.sonar.api.config.Settings; +import org.sonar.batch.cpd.index.SonarDuplicationsIndex; +import org.sonar.batch.index.BatchComponentCache; +import org.sonar.duplications.block.Block; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; + +public class JavaCpdIndexerTest { + private static final String JAVA = "java"; + + @Mock + private SonarDuplicationsIndex index; + + @Captor + private ArgumentCaptor> blockCaptor; + + private Settings settings; + private JavaCpdIndexer engine; + private DefaultInputFile file; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Before + public void setUp() throws IOException { + MockitoAnnotations.initMocks(this); + + File baseDir = temp.newFolder(); + DefaultFileSystem fs = new DefaultFileSystem(baseDir); + file = new DefaultInputFile("foo", "src/ManyStatements.java").setLanguage(JAVA); + fs.add(file); + BatchComponentCache batchComponentCache = new BatchComponentCache(); + batchComponentCache.add(org.sonar.api.resources.File.create("src/Foo.java").setEffectiveKey("foo:src/ManyStatements.java"), null).setInputComponent(file); + File ioFile = file.file(); + FileUtils.copyURLToFile(this.getClass().getResource("ManyStatements.java"), ioFile); + + settings = new Settings(); + engine = new JavaCpdIndexer(fs, settings, index); + } + + @Test + public void languageSupported() { + JavaCpdIndexer engine = new JavaCpdIndexer(mock(FileSystem.class), new Settings(), index); + assertThat(engine.isLanguageSupported(JAVA)).isTrue(); + assertThat(engine.isLanguageSupported("php")).isFalse(); + } + + @Test + public void testExclusions() { + settings.setProperty(CoreProperties.CPD_EXCLUSIONS, "**"); + engine.index(JAVA); + verifyZeroInteractions(index); + } + + @Test + public void testJavaIndexing() throws Exception { + engine.index(JAVA); + + verify(index).insert(eq(file), blockCaptor.capture()); + List blockList = blockCaptor.getValue(); + + assertThat(blockList).hasSize(26); + } +} diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java index bb41a88d7e6..83158d89d1b 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java @@ -80,12 +80,80 @@ public class CpdMediumTest { tester.stop(); } + @Test + public void testCrossModuleDuplications() throws IOException { + builder.put("sonar.modules", "module1,module2") + .put("sonar.cpd.xoo.minimumTokens", "10") + .put("sonar.verbose", "true"); + + // module 1 + builder.put("module1.sonar.projectKey", "module1"); + builder.put("module1.sonar.projectName", "Module 1"); + builder.put("module1.sonar.sources", "."); + + // module2 + builder.put("module2.sonar.projectKey", "module2"); + builder.put("module2.sonar.projectName", "Module 2"); + builder.put("module2.sonar.sources", "."); + + File module1Dir = new File(baseDir, "module1"); + File module2Dir = new File(baseDir, "module2"); + + module1Dir.mkdir(); + module2Dir.mkdir(); + + String duplicatedStuff = "Sample xoo\ncontent\n" + + "foo\nbar\ntoto\ntiti\n" + + "foo\nbar\ntoto\ntiti\n" + + "bar\ntoto\ntiti\n" + + "foo\nbar\ntoto\ntiti"; + + // create duplicated file in both modules + File xooFile1 = new File(module1Dir, "sample1.xoo"); + FileUtils.write(xooFile1, duplicatedStuff); + + File xooFile2 = new File(module2Dir, "sample2.xoo"); + FileUtils.write(xooFile2, duplicatedStuff); + + TaskResult result = tester.newTask().properties(builder.build()).start(); + + assertThat(result.inputFiles()).hasSize(2); + + InputFile inputFile1 = result.inputFile("sample1.xoo"); + InputFile inputFile2 = result.inputFile("sample2.xoo"); + + // One clone group on each file + List duplicationGroupsFile1 = result.duplicationsFor(inputFile1); + assertThat(duplicationGroupsFile1).hasSize(1); + + org.sonar.batch.protocol.output.BatchReport.Duplication cloneGroupFile1 = duplicationGroupsFile1.get(0); + assertThat(cloneGroupFile1.getOriginPosition().getStartLine()).isEqualTo(1); + assertThat(cloneGroupFile1.getOriginPosition().getEndLine()).isEqualTo(17); + assertThat(cloneGroupFile1.getDuplicateList()).hasSize(1); + assertThat(cloneGroupFile1.getDuplicate(0).getOtherFileRef()).isEqualTo(result.getReportComponent(((DefaultInputFile) inputFile2).key()).getRef()); + + List duplicationGroupsFile2 = result.duplicationsFor(inputFile2); + assertThat(duplicationGroupsFile2).hasSize(1); + + org.sonar.batch.protocol.output.BatchReport.Duplication cloneGroupFile2 = duplicationGroupsFile2.get(0); + assertThat(cloneGroupFile2.getOriginPosition().getStartLine()).isEqualTo(1); + assertThat(cloneGroupFile2.getOriginPosition().getEndLine()).isEqualTo(17); + assertThat(cloneGroupFile2.getDuplicateList()).hasSize(1); + assertThat(cloneGroupFile2.getDuplicate(0).getOtherFileRef()).isEqualTo(result.getReportComponent(((DefaultInputFile) inputFile1).key()).getRef()); + + assertThat(result.duplicationBlocksFor(inputFile1)).isEmpty(); + } + @Test public void testCrossFileDuplications() throws IOException { File srcDir = new File(baseDir, "src"); srcDir.mkdir(); - String duplicatedStuff = "Sample xoo\ncontent\nfoo\nbar\ntoto\ntiti\nfoo\nbar\ntoto\ntiti\nbar\ntoto\ntiti\nfoo\nbar\ntoto\ntiti"; + String duplicatedStuff = "Sample xoo\ncontent\n" + + "foo\nbar\ntoto\ntiti\n" + + "foo\nbar\ntoto\ntiti\n" + + "bar\ntoto\ntiti\n" + + "foo\nbar\ntoto\ntiti"; File xooFile1 = new File(srcDir, "sample1.xoo"); FileUtils.write(xooFile1, duplicatedStuff); @@ -103,8 +171,6 @@ public class CpdMediumTest { assertThat(result.inputFiles()).hasSize(2); - Map> allMeasures = result.allMeasures(); - InputFile inputFile1 = result.inputFile("src/sample1.xoo"); InputFile inputFile2 = result.inputFile("src/sample2.xoo"); diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/index/CloneIndex.java b/sonar-duplications/src/main/java/org/sonar/duplications/index/CloneIndex.java index 357fca44aa8..7bcf94b914a 100644 --- a/sonar-duplications/src/main/java/org/sonar/duplications/index/CloneIndex.java +++ b/sonar-duplications/src/main/java/org/sonar/duplications/index/CloneIndex.java @@ -20,9 +20,11 @@ package org.sonar.duplications.index; import java.util.Collection; +import java.util.Iterator; import org.sonar.duplications.block.Block; import org.sonar.duplications.block.ByteArray; +import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks; public interface CloneIndex { @@ -45,4 +47,9 @@ public interface CloneIndex { */ void insert(Block block); + /** + * Iterators through the resources, providing the list of blocks for each resource. + */ + Iterator iterator(); + } diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/index/MemoryCloneIndex.java b/sonar-duplications/src/main/java/org/sonar/duplications/index/MemoryCloneIndex.java index 393a8e7d1dd..1b454bf6eff 100644 --- a/sonar-duplications/src/main/java/org/sonar/duplications/index/MemoryCloneIndex.java +++ b/sonar-duplications/src/main/java/org/sonar/duplications/index/MemoryCloneIndex.java @@ -23,8 +23,10 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import org.sonar.duplications.block.Block; import org.sonar.duplications.block.ByteArray; +import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks; import java.util.Collection; +import java.util.Iterator; public class MemoryCloneIndex implements CloneIndex { @@ -47,4 +49,9 @@ public class MemoryCloneIndex implements CloneIndex { byHash.put(block.getBlockHash(), block); } + @Override + public Iterator iterator() { + throw new UnsupportedOperationException(); + } + } diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/index/PackedMemoryCloneIndex.java b/sonar-duplications/src/main/java/org/sonar/duplications/index/PackedMemoryCloneIndex.java index f0257cef0e2..5681acf1f55 100644 --- a/sonar-duplications/src/main/java/org/sonar/duplications/index/PackedMemoryCloneIndex.java +++ b/sonar-duplications/src/main/java/org/sonar/duplications/index/PackedMemoryCloneIndex.java @@ -21,11 +21,16 @@ package org.sonar.duplications.index; import java.util.ArrayList; import java.util.Collection; +import java.util.Iterator; import java.util.List; +import java.util.NoSuchElementException; + import org.sonar.duplications.block.Block; import org.sonar.duplications.block.ByteArray; import org.sonar.duplications.utils.FastStringComparator; +import javax.annotation.Nullable; + /** * Provides an index optimized by memory. *

@@ -104,31 +109,105 @@ public class PackedMemoryCloneIndex extends AbstractCloneIndex { List result = new ArrayList<>(); int realIndex = resourceIdsIndex[index]; while (index < size && FastStringComparator.INSTANCE.compare(resourceIds[realIndex], resourceId) == 0) { - // extract block (note that there is no need to extract resourceId) - int offset = realIndex * blockInts; + result.add(getBlock(realIndex, resourceId)); + + index++; + realIndex = resourceIdsIndex[index]; + } + return result; + } + + private Block createBlock(int index, String resourceId, @Nullable ByteArray byteHash) { + int offset = index * blockInts; + ByteArray blockHash; + + if (byteHash == null) { int[] hash = new int[hashInts]; for (int j = 0; j < hashInts; j++) { hash[j] = blockData[offset++]; } - int indexInFile = blockData[offset++]; - int firstLineNumber = blockData[offset++]; - int lastLineNumber = blockData[offset++]; - int startUnit = blockData[offset++]; - int endUnit = blockData[offset]; - - Block block = blockBuilder - .setResourceId(resourceId) - .setBlockHash(new ByteArray(hash)) - .setIndexInFile(indexInFile) - .setLines(firstLineNumber, lastLineNumber) - .setUnit(startUnit, endUnit) - .build(); - result.add(block); + blockHash = new ByteArray(hash); + } else { + blockHash = byteHash; + offset += hashInts; + } - index++; - realIndex = resourceIdsIndex[index]; + int indexInFile = blockData[offset++]; + int firstLineNumber = blockData[offset++]; + int lastLineNumber = blockData[offset++]; + int startUnit = blockData[offset++]; + int endUnit = blockData[offset]; + + return blockBuilder + .setResourceId(resourceId) + .setBlockHash(blockHash) + .setIndexInFile(indexInFile) + .setLines(firstLineNumber, lastLineNumber) + .setUnit(startUnit, endUnit) + .build(); + } + + private Block getBlock(int index, String resourceId) { + return createBlock(index, resourceId, null); + } + + private class ResourceIterator implements Iterator { + private int index = 0; + + @Override + public boolean hasNext() { + return index < size; } - return result; + + @Override + public ResourceBlocks next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + String resourceId = resourceIds[resourceIdsIndex[index]]; + List blocks = new ArrayList<>(); + + // while we are at the same resource, keep going + do { + blocks.add(getBlock(resourceIdsIndex[index], resourceId)); + index++; + } while (hasNext() && FastStringComparator.INSTANCE.compare(resourceIds[resourceIdsIndex[index]], resourceId) == 0); + + return new ResourceBlocks(resourceId, blocks); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } + + public static class ResourceBlocks { + private Collection blocks; + private String resourceId; + + public ResourceBlocks(String resourceId, Collection blocks) { + this.resourceId = resourceId; + this.blocks = blocks; + } + + public Collection blocks() { + return blocks; + } + + public String resourceId() { + return resourceId; + } + } + + /** + * {@inheritDoc} + */ + @Override + public Iterator iterator() { + ensureSorted(); + return new ResourceIterator(); } /** @@ -154,21 +233,7 @@ public class PackedMemoryCloneIndex extends AbstractCloneIndex { while (index < size && !isLessByHash(size, index)) { // extract block (note that there is no need to extract hash) String resourceId = resourceIds[index]; - offset = index * blockInts + hashInts; - int indexInFile = blockData[offset++]; - int firstLineNumber = blockData[offset++]; - int lastLineNumber = blockData[offset++]; - int startUnit = blockData[offset++]; - int endUnit = blockData[offset]; - - Block block = blockBuilder - .setResourceId(resourceId) - .setBlockHash(sequenceHash) - .setIndexInFile(indexInFile) - .setLines(firstLineNumber, lastLineNumber) - .setUnit(startUnit, endUnit) - .build(); - result.add(block); + result.add(createBlock(index, resourceId, sequenceHash)); index++; } return result; diff --git a/sonar-duplications/src/test/java/org/sonar/duplications/index/PackedMemoryCloneIndexTest.java b/sonar-duplications/src/test/java/org/sonar/duplications/index/PackedMemoryCloneIndexTest.java index cb91fe37b6f..0812e1eb767 100644 --- a/sonar-duplications/src/test/java/org/sonar/duplications/index/PackedMemoryCloneIndexTest.java +++ b/sonar-duplications/src/test/java/org/sonar/duplications/index/PackedMemoryCloneIndexTest.java @@ -23,9 +23,13 @@ import org.junit.Before; import org.junit.Test; import org.sonar.duplications.block.Block; import org.sonar.duplications.block.ByteArray; +import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks; +import java.util.ArrayList; import java.util.Collection; +import java.util.Iterator; +import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; import static org.junit.Assert.assertThat; @@ -76,6 +80,34 @@ public class PackedMemoryCloneIndexTest { assertThat(block.getBlockHash(), sameInstance(requestedHash)); } } + + @Test + public void iterate() { + index.insert(newBlock("a", 1)); + index.insert(newBlock("c", 1)); + index.insert(newBlock("b", 1)); + index.insert(newBlock("c", 2)); + index.insert(newBlock("a", 2)); + + Iterator it = index.iterator(); + + ArrayList resourcesBlocks = new ArrayList<>(); + + while(it.hasNext()) { + resourcesBlocks.add(it.next()); + } + + assertThat(resourcesBlocks).hasSize(3); + + assertThat(resourcesBlocks.get(0).resourceId()).isEqualTo("a"); + assertThat(resourcesBlocks.get(1).resourceId()).isEqualTo("b"); + assertThat(resourcesBlocks.get(2).resourceId()).isEqualTo("c"); + + assertThat(resourcesBlocks.get(0).blocks()).hasSize(2); + assertThat(resourcesBlocks.get(1).blocks()).hasSize(1); + assertThat(resourcesBlocks.get(2).blocks()).hasSize(2); + + } /** * Given: index with initial capacity 1. -- 2.39.5