--- /dev/null
+sonar.projectKey=module1
+sonar.projectName=Module 1
+sonar.projectVersion=1.0-SNAPSHOT
+sonar.sources=src/main/xoo
+sonar.language=xoo
--- /dev/null
+package sample;
+
+public class File1 {
+
+ public File1() {
+ }
+
+ public void test() {
+ char[] charList = new char[30];
+ for (int i = 0; i < 10; i++) {
+ charList[i] = 'a';
+ }
+ for (int i = 0; i < 10; i++) {
+ charList[i] = 'a';
+ }
+ int intergerToBeIncremented = 0;
+ while (intergerToBeIncremented < 100) {
+ intergerToBeIncremented++;
+ }
+ int intergerToBeIncremented2 = 0;
+ while (intergerToBeIncremented2 < 100) {
+ intergerToBeIncremented2++;
+ }
+ String temp = "";
+ for (int i=0; i<10; i++){
+ temp += "say something"+i;
+ }
+ for (int i=0; i<20; i++){
+ temp += "say nothing"+i;
+ }
+ for (int i=0; i<30; i++){
+ temp += "always say nothing"+i;
+ }
+ }
+}
--- /dev/null
+package sample;
+
+public class File1 {
+
+ public File1() {
+ }
+
+ public void otherMethod() {
+ String temp = "";
+ for (int i=0; i<10; i++){
+ temp += "say something"+i;
+ int nothing = 0;
+ }
+ for (int i=0; i<20; i++){
+ temp += "say nothing"+i;
+ int nothing = 1;
+ }
+ for (int i=0; i<30; i++){
+ temp += "always say nothing"+i;
+ int nothing = 2;
+ }
+ }
+}
--- /dev/null
+sonar.projectKey=module2
+sonar.projectName=Module 2
+sonar.projectVersion=1.0-SNAPSHOT
+sonar.sources=src/main/xoo
+sonar.language=xoo
--- /dev/null
+package sample;
+
+public class File1 {
+
+ public File1() {
+ }
+
+ public void test2() {
+ char[] charList = new char[30];
+ for (int i = 0; i < 10; i++) {
+ charList[i] = 'a';
+ }
+ for (int i = 0; i < 10; i++) {
+ charList[i] = 'a';
+ }
+ int intergerToBeIncremented = 0;
+ while (intergerToBeIncremented < 100) {
+ intergerToBeIncremented++;
+ }
+ int intergerToBeIncremented2 = 0;
+ while (intergerToBeIncremented2 < 100) {
+ intergerToBeIncremented2++;
+ }
+ String temp = "";
+ for (int i=0; i<10; i++){
+ temp += "say something"+i;
+ }
+ for (int i=0; i<20; i++){
+ temp += "say nothing"+i;
+ }
+ for (int i=0; i<30; i++){
+ temp += "always say nothing"+i;
+ }
+ }
+}
--- /dev/null
+sonar.projectKey=cross-module
+sonar.projectName=Cross Module Duplication
+sonar.projectVersion=1.0-SNAPSHOT
+sonar.modules=module1,module2
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package it.duplication;
+
+import com.google.common.collect.ImmutableMap;
+import com.sonar.orchestrator.Orchestrator;
+import com.sonar.orchestrator.build.SonarRunner;
+import com.sonar.orchestrator.locator.FileLocation;
+import it.Category4Suite;
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.wsclient.services.Resource;
+import org.sonar.wsclient.services.ResourceQuery;
+import util.ItUtils;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class CrossModuleDuplicationsTest {
+ private static final String PROJECT_KEY = "cross-module";
+ private static final String PROJECT_DIR = "duplications/" + PROJECT_KEY;
+ private File projectDir;
+
+ @ClassRule
+ public static Orchestrator orchestrator = Category4Suite.ORCHESTRATOR;
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ @BeforeClass
+ public static void analyzeProjects() {
+
+ }
+
+ @Before
+ public void setUpProject() throws IOException {
+ orchestrator.resetData();
+ orchestrator.getServer().restoreProfile(FileLocation.ofClasspath("/duplication/xoo-duplication-profile.xml"));
+
+ FileUtils.copyDirectory(ItUtils.projectDir(PROJECT_DIR), temp.getRoot());
+ projectDir = temp.getRoot();
+ }
+
+ @Test
+ public void testDuplications() throws IOException {
+ analyzeProject(projectDir, PROJECT_KEY, true);
+ verifyDuplicationMeasures(PROJECT_KEY, 2, 54, 2, 56.3);
+
+ // File1 is the one duplicated in both modules
+ verifyDuplicationMeasures(PROJECT_KEY + ":module1:src/main/xoo/sample/File1.xoo", 1, 27, 1, 75);
+ verifyDuplicationMeasures(PROJECT_KEY + ":module2:src/main/xoo/sample/File1.xoo", 1, 27, 1, 75);
+ }
+
+ @Test
+ // SONAR-6184
+ public void testGhostDuplication() throws IOException {
+ analyzeProject(projectDir, PROJECT_KEY, true);
+
+ verifyDuplicationMeasures(PROJECT_KEY + ":module1", 1, 27, 1, 45);
+ verifyDuplicationMeasures(PROJECT_KEY + ":module2", 1, 27, 1, 75);
+
+ // move File2 from module1 to module2
+ File src = FileUtils.getFile(projectDir, "module1", "src", "main", "xoo", "sample", "File2.xoo");
+ File dst = FileUtils.getFile(projectDir, "module2", "src", "main", "xoo", "sample", "File2.xoo");
+ FileUtils.moveFile(src, dst);
+
+ src = new File(src.getParentFile(), "File2.xoo.measures");
+ dst = new File(dst.getParentFile(), "File2.xoo.measures");
+ FileUtils.moveFile(src, dst);
+
+ // duplication should remain unchanged (except for % of duplication)
+ analyzeProject(projectDir, PROJECT_KEY, false);
+ verifyDuplicationMeasures(PROJECT_KEY + ":module1", 1, 27, 1, 75);
+ verifyDuplicationMeasures(PROJECT_KEY + ":module2", 1, 27, 1, 45);
+ }
+
+ @Test
+ // SONAR-6184
+ public void testDuplicationFix() throws IOException {
+ analyzeProject(projectDir, PROJECT_KEY, true);
+
+ verifyDuplicationMeasures(PROJECT_KEY + ":module1", 1, 27, 1, 45);
+ verifyDuplicationMeasures(PROJECT_KEY + ":module2", 1, 27, 1, 75);
+
+ // remove File1 from module1
+ File f1 = FileUtils.getFile(projectDir, "module1", "src", "main", "xoo", "sample", "File1.xoo");
+ File f1m = FileUtils.getFile(projectDir, "module2", "src", "main", "xoo", "sample", "File1.xoo.measures");
+ f1.delete();
+ f1m.delete();
+
+ // duplication should be 0
+ analyzeProject(projectDir, PROJECT_KEY, false);
+ verifyDuplicationMeasures(PROJECT_KEY + ":module1", 0, 0, 0, 0);
+ verifyDuplicationMeasures(PROJECT_KEY + ":module2", 0, 0, 0, 0);
+ }
+
+ private static SonarRunner analyzeProject(File projectDir, String projectKey, boolean create, String... additionalProperties) {
+ if (create) {
+ orchestrator.getServer().provisionProject(projectKey, projectKey);
+ orchestrator.getServer().associateProjectToQualityProfile(projectKey, "xoo", "xoo-duplication-profile");
+ }
+
+ SonarRunner sonarRunner = SonarRunner.create(projectDir);
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+
+ for (int i = 0; i < additionalProperties.length; i += 2) {
+ builder.put(additionalProperties[i], additionalProperties[i + 1]);
+ }
+ SonarRunner scan = sonarRunner.setDebugLogs(true).setProperties(builder.build());
+ orchestrator.executeBuild(scan);
+ return scan;
+ }
+
+ private static void verifyDuplicationMeasures(String componentKey, int duplicatedBlocks, int duplicatedLines, int duplicatedFiles, double duplicatedLinesDensity) {
+ Resource file = getComponent(componentKey);
+ assertThat(file.getMeasureValue("duplicated_blocks").intValue()).isEqualTo(duplicatedBlocks);
+ assertThat(file.getMeasureValue("duplicated_lines").intValue()).isEqualTo(duplicatedLines);
+ assertThat(file.getMeasureValue("duplicated_files").intValue()).isEqualTo(duplicatedFiles);
+ assertThat(file.getMeasureValue("duplicated_lines_density")).isEqualTo(duplicatedLinesDensity);
+ }
+
+ private static Resource getComponent(String key) {
+ Resource component = orchestrator.getServer().getWsClient()
+ .find(ResourceQuery.createForMetrics(key, "duplicated_lines", "duplicated_blocks", "duplicated_files", "duplicated_lines_density"));
+ assertThat(component).isNotNull();
+ return component;
+ }
+}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-package org.sonar.batch.cpd;
-
-import com.google.common.base.Function;
-import java.util.List;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.utils.log.Logger;
-import org.sonar.api.utils.log.Loggers;
-import org.sonar.batch.index.BatchComponent;
-import org.sonar.batch.index.BatchComponentCache;
-import org.sonar.batch.protocol.output.BatchReport;
-import org.sonar.batch.protocol.output.BatchReport.Duplicate;
-import org.sonar.batch.protocol.output.BatchReport.Duplication;
-import org.sonar.batch.report.ReportPublisher;
-import org.sonar.duplications.index.CloneGroup;
-import org.sonar.duplications.index.ClonePart;
-
-import static com.google.common.collect.FluentIterable.from;
-
-public abstract class AbstractCpdEngine extends CpdEngine {
-
- private static final Logger LOG = Loggers.get(AbstractCpdEngine.class);
-
- static final int MAX_CLONE_GROUP_PER_FILE = 100;
- static final int MAX_CLONE_PART_PER_GROUP = 100;
-
- private final ReportPublisher publisher;
- private final BatchComponentCache batchComponentCache;
-
- public AbstractCpdEngine(ReportPublisher publisher, BatchComponentCache batchComponentCache) {
- this.publisher = publisher;
- this.batchComponentCache = batchComponentCache;
- }
-
- protected final void saveDuplications(final InputFile inputFile, List<CloneGroup> duplications) {
- if (duplications.size() > MAX_CLONE_GROUP_PER_FILE) {
- LOG.warn("Too many duplication groups on file " + inputFile.relativePath() + ". Keep only the first " + MAX_CLONE_GROUP_PER_FILE + " groups.");
- }
- final BatchComponent component = batchComponentCache.get(inputFile);
- Iterable<org.sonar.batch.protocol.output.BatchReport.Duplication> reportDuplications = from(duplications)
- .limit(MAX_CLONE_GROUP_PER_FILE)
- .transform(
- new Function<CloneGroup, BatchReport.Duplication>() {
- private final BatchReport.Duplication.Builder dupBuilder = BatchReport.Duplication.newBuilder();
- private final BatchReport.Duplicate.Builder blockBuilder = BatchReport.Duplicate.newBuilder();
-
- @Override
- public BatchReport.Duplication apply(CloneGroup input) {
- return toReportDuplication(component, inputFile, dupBuilder, blockBuilder, input);
- }
-
- });
- publisher.getWriter().writeComponentDuplications(component.batchId(), reportDuplications);
- }
-
- private Duplication toReportDuplication(BatchComponent component, InputFile inputFile, Duplication.Builder dupBuilder, Duplicate.Builder blockBuilder, CloneGroup input) {
- dupBuilder.clear();
- ClonePart originBlock = input.getOriginPart();
- blockBuilder.clear();
- dupBuilder.setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(originBlock.getStartLine())
- .setEndLine(originBlock.getEndLine())
- .build());
- int clonePartCount = 0;
- for (ClonePart duplicate : input.getCloneParts()) {
- if (!duplicate.equals(originBlock)) {
- clonePartCount++;
- if (clonePartCount > MAX_CLONE_PART_PER_GROUP) {
- LOG.warn("Too many duplication references on file " + inputFile.relativePath() + " for block at line " + originBlock.getStartLine() + ". Keep only the first "
- + MAX_CLONE_PART_PER_GROUP + " references.");
- break;
- }
- blockBuilder.clear();
- String componentKey = duplicate.getResourceId();
- if (!component.key().equals(componentKey)) {
- BatchComponent sameProjectComponent = batchComponentCache.get(componentKey);
- blockBuilder.setOtherFileRef(sameProjectComponent.batchId());
- }
- dupBuilder.addDuplicate(blockBuilder
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(duplicate.getStartLine())
- .setEndLine(duplicate.getEndLine())
- .build())
- .build());
- }
- }
- return dupBuilder.build();
- }
-
-}
private CpdComponents() {
}
- public static List all() {
+ public static List<Class<? extends Object>> all() {
return ImmutableList.of(
CpdSensor.class,
CpdMappings.class,
- JavaCpdEngine.class,
- DefaultCpdEngine.class);
+ JavaCpdIndexer.class,
+ DefaultCpdIndexer.class);
}
}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.batch.cpd;
-
-import org.slf4j.Logger;
-import org.sonar.api.batch.BatchSide;
-import org.sonar.api.batch.sensor.SensorContext;
-
-@BatchSide
-public abstract class CpdEngine {
-
- abstract boolean isLanguageSupported(String language);
-
- abstract void analyse(String language, SensorContext context);
-
- protected void logExclusions(String[] exclusions, Logger logger) {
- if (exclusions.length > 0) {
- StringBuilder message = new StringBuilder("Copy-paste detection exclusions:");
- for (String exclusion : exclusions) {
- message.append("\n ");
- message.append(exclusion);
- }
-
- logger.info(message.toString());
- }
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName();
- }
-
-}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.batch.cpd;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.config.Settings;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.index.BatchComponent;
+import org.sonar.batch.index.BatchComponentCache;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.batch.protocol.output.BatchReport.Duplicate;
+import org.sonar.batch.protocol.output.BatchReport.Duplication;
+import org.sonar.batch.report.ReportPublisher;
+import org.sonar.duplications.block.Block;
+import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm;
+import org.sonar.duplications.index.CloneGroup;
+import org.sonar.duplications.index.ClonePart;
+import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import static com.google.common.collect.FluentIterable.from;
+
+/**
+ * Runs on the root module, at the end of the project analysis.
+ * It executes copy paste detection involving all files of all modules, which were indexed during sensors execution for each module
+ * by {@link CpdSensor). The sensor is responsible for handling exclusions and block sizes.
+ */
+public class CpdExecutor {
+ private static final Logger LOG = Loggers.get(CpdExecutor.class);
+ static final int MAX_CLONE_GROUP_PER_FILE = 100;
+ static final int MAX_CLONE_PART_PER_GROUP = 100;
+
+ private final SonarDuplicationsIndex index;
+ private final ReportPublisher publisher;
+ private final BatchComponentCache batchComponentCache;
+ private final Settings settings;
+
+ public CpdExecutor(Settings settings, SonarDuplicationsIndex index, ReportPublisher publisher, BatchComponentCache batchComponentCache) {
+ this.settings = settings;
+ this.index = index;
+ this.publisher = publisher;
+ this.batchComponentCache = batchComponentCache;
+ }
+
+ public void execute() {
+ Iterator<ResourceBlocks> it = index.iterator();
+
+ while (it.hasNext()) {
+ ResourceBlocks resourceBlocks = it.next();
+ runCpdAnalysis(resourceBlocks.resourceId(), resourceBlocks.blocks());
+ }
+ }
+
+ private void runCpdAnalysis(String resource, Collection<Block> fileBlocks) {
+ LOG.debug("Detection of duplications for {}", resource);
+
+ BatchComponent component = batchComponentCache.get(resource);
+ if (component == null) {
+ LOG.error("Resource not found in component cache: {}. Skipping CPD computation for it", resource);
+ return;
+ }
+
+ List<CloneGroup> duplications;
+ try {
+ duplications = SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks);
+ } catch (Exception e) {
+ throw new IllegalStateException("Fail during detection of duplication for " + resource, e);
+ }
+
+ InputFile inputFile = (InputFile) component.inputComponent();
+ Predicate<CloneGroup> minimumTokensPredicate = DuplicationPredicates.numberOfUnitsNotLessThan(getMinimumTokens(inputFile.language()));
+ List<CloneGroup> filtered = from(duplications).filter(minimumTokensPredicate).toList();
+
+ saveDuplications(component, filtered);
+ }
+
+ @VisibleForTesting
+ int getMinimumTokens(String languageKey) {
+ int minimumTokens = settings.getInt("sonar.cpd." + languageKey + ".minimumTokens");
+ if (minimumTokens == 0) {
+ minimumTokens = 100;
+ }
+
+ return minimumTokens;
+ }
+
+ @VisibleForTesting
+ final void saveDuplications(final BatchComponent component, List<CloneGroup> duplications) {
+ if (duplications.size() > MAX_CLONE_GROUP_PER_FILE) {
+ LOG.warn("Too many duplication groups on file " + component.inputComponent() + ". Keep only the first " + MAX_CLONE_GROUP_PER_FILE +
+ " groups.");
+ }
+ Iterable<org.sonar.batch.protocol.output.BatchReport.Duplication> reportDuplications = from(duplications)
+ .limit(MAX_CLONE_GROUP_PER_FILE)
+ .transform(
+ new Function<CloneGroup, BatchReport.Duplication>() {
+ private final BatchReport.Duplication.Builder dupBuilder = BatchReport.Duplication.newBuilder();
+ private final BatchReport.Duplicate.Builder blockBuilder = BatchReport.Duplicate.newBuilder();
+
+ @Override
+ public BatchReport.Duplication apply(CloneGroup input) {
+ return toReportDuplication(component, dupBuilder, blockBuilder, input);
+ }
+
+ });
+ publisher.getWriter().writeComponentDuplications(component.batchId(), reportDuplications);
+ }
+
+ private Duplication toReportDuplication(BatchComponent component, Duplication.Builder dupBuilder, Duplicate.Builder blockBuilder, CloneGroup input) {
+ dupBuilder.clear();
+ ClonePart originBlock = input.getOriginPart();
+ blockBuilder.clear();
+ dupBuilder.setOriginPosition(BatchReport.TextRange.newBuilder()
+ .setStartLine(originBlock.getStartLine())
+ .setEndLine(originBlock.getEndLine())
+ .build());
+ int clonePartCount = 0;
+ for (ClonePart duplicate : input.getCloneParts()) {
+ if (!duplicate.equals(originBlock)) {
+ clonePartCount++;
+ if (clonePartCount > MAX_CLONE_PART_PER_GROUP) {
+ LOG.warn("Too many duplication references on file " + component.inputComponent() + " for block at line " +
+ originBlock.getStartLine() + ". Keep only the first "
+ + MAX_CLONE_PART_PER_GROUP + " references.");
+ break;
+ }
+ blockBuilder.clear();
+ String componentKey = duplicate.getResourceId();
+ if (!component.key().equals(componentKey)) {
+ BatchComponent sameProjectComponent = batchComponentCache.get(componentKey);
+ blockBuilder.setOtherFileRef(sameProjectComponent.batchId());
+ }
+ dupBuilder.addDuplicate(blockBuilder
+ .setRange(BatchReport.TextRange.newBuilder()
+ .setStartLine(duplicate.getStartLine())
+ .setEndLine(duplicate.getEndLine())
+ .build())
+ .build());
+ }
+ }
+ return dupBuilder.build();
+ }
+}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.batch.cpd;
+
+import org.slf4j.Logger;
+import org.sonar.api.batch.BatchSide;
+
+@BatchSide
+public abstract class CpdIndexer {
+
+ abstract boolean isLanguageSupported(String language);
+
+ abstract void index(String language);
+
+ protected void logExclusions(String[] exclusions, Logger logger) {
+ if (exclusions.length > 0) {
+ StringBuilder message = new StringBuilder("Copy-paste detection exclusions:");
+ for (String exclusion : exclusions) {
+ message.append("\n ");
+ message.append(exclusion);
+ }
+
+ logger.info(message.toString());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+
+}
private static final Logger LOG = LoggerFactory.getLogger(CpdSensor.class);
- private CpdEngine sonarEngine;
- private CpdEngine sonarBridgeEngine;
+ private CpdIndexer sonarEngine;
+ private CpdIndexer sonarBridgeEngine;
private Settings settings;
private FileSystem fs;
- public CpdSensor(JavaCpdEngine sonarEngine, DefaultCpdEngine sonarBridgeEngine, Settings settings, FileSystem fs) {
+ public CpdSensor(JavaCpdIndexer sonarEngine, DefaultCpdIndexer sonarBridgeEngine, Settings settings, FileSystem fs) {
this.sonarEngine = sonarEngine;
this.sonarBridgeEngine = sonarBridgeEngine;
this.settings = settings;
}
@VisibleForTesting
- CpdEngine getEngine(String language) {
+ CpdIndexer getEngine(String language) {
if (sonarEngine.isLanguageSupported(language)) {
return sonarEngine;
}
continue;
}
- CpdEngine engine = getEngine(language);
+ CpdIndexer engine = getEngine(language);
if (!engine.isLanguageSupported(language)) {
LOG.debug("Detection of duplicated code is not supported for {}", language);
continue;
}
LOG.info("{} is used for {}", engine, language);
- engine.analyse(language, context);
+ engine.index(language);
}
}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.batch.cpd;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Lists;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.CpdMapping;
-import org.sonar.api.batch.fs.FilePredicates;
-import org.sonar.api.batch.fs.FileSystem;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.config.Settings;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
-import org.sonar.batch.index.BatchComponentCache;
-import org.sonar.batch.report.ReportPublisher;
-import org.sonar.duplications.block.Block;
-import org.sonar.duplications.index.CloneGroup;
-import org.sonar.duplications.internal.pmd.TokenizerBridge;
-
-import static com.google.common.collect.FluentIterable.from;
-
-public class DefaultCpdEngine extends AbstractCpdEngine {
-
- private static final Logger LOG = LoggerFactory.getLogger(DefaultCpdEngine.class);
-
- /**
- * Limit of time to analyse one file (in seconds).
- */
- private static final int TIMEOUT = 5 * 60;
-
- private final CpdMappings mappings;
- private final FileSystem fs;
- private final Settings settings;
- private final ReportPublisher publisher;
- private final BatchComponentCache batchComponentCache;
-
- public DefaultCpdEngine(CpdMappings mappings, FileSystem fs, Settings settings, ReportPublisher publisher, BatchComponentCache batchComponentCache) {
- super(publisher, batchComponentCache);
- this.mappings = mappings;
- this.fs = fs;
- this.settings = settings;
- this.publisher = publisher;
- this.batchComponentCache = batchComponentCache;
- }
-
- @Override
- public boolean isLanguageSupported(String language) {
- return true;
- }
-
- @Override
- public void analyse(String languageKey, SensorContext context) {
- CpdMapping mapping = mappings.getMapping(languageKey);
- if (mapping == null) {
- LOG.debug("No CpdMapping for language " + languageKey);
- return;
- }
-
- String[] cpdExclusions = settings.getStringArray(CoreProperties.CPD_EXCLUSIONS);
- logExclusions(cpdExclusions, LOG);
- FilePredicates p = fs.predicates();
- List<InputFile> sourceFiles = Lists.newArrayList(fs.inputFiles(p.and(
- p.hasType(InputFile.Type.MAIN),
- p.hasLanguage(languageKey),
- p.doesNotMatchPathPatterns(cpdExclusions))));
- if (sourceFiles.isEmpty()) {
- return;
- }
-
- // Create index
- SonarDuplicationsIndex index = new SonarDuplicationsIndex(publisher, batchComponentCache, settings);
- populateIndex(languageKey, sourceFiles, mapping, index);
-
- // Detect
- runCpdAnalysis(languageKey, context, sourceFiles, index);
- }
-
- private void runCpdAnalysis(String languageKey, SensorContext context, List<InputFile> sourceFiles, SonarDuplicationsIndex index) {
- Predicate<CloneGroup> minimumTokensPredicate = DuplicationPredicates.numberOfUnitsNotLessThan(getMinimumTokens(languageKey));
-
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- try {
- for (InputFile inputFile : sourceFiles) {
- LOG.debug("Detection of duplications for {}", inputFile);
- String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
- Collection<Block> fileBlocks = index.getByInputFile(inputFile, resourceEffectiveKey);
-
- List<CloneGroup> filtered;
- try {
- List<CloneGroup> duplications = executorService.submit(new JavaCpdEngine.Task(index, fileBlocks)).get(TIMEOUT, TimeUnit.SECONDS);
- filtered = from(duplications)
- .filter(minimumTokensPredicate)
- .toList();
- } catch (TimeoutException e) {
- filtered = Collections.emptyList();
- LOG.warn("Timeout during detection of duplications for " + inputFile, e);
- } catch (InterruptedException | ExecutionException e) {
- throw new IllegalStateException("Fail during detection of duplication for " + inputFile, e);
- }
-
- saveDuplications(inputFile, filtered);
- }
- } finally {
- executorService.shutdown();
- }
- }
-
- private void populateIndex(String languageKey, List<InputFile> sourceFiles, CpdMapping mapping, SonarDuplicationsIndex index) {
- TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), fs.encoding().name(), getBlockSize(languageKey));
- for (InputFile inputFile : sourceFiles) {
- LOG.debug("Populating index from {}", inputFile);
- String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
- List<Block> blocks = bridge.chunk(resourceEffectiveKey, inputFile.file());
- index.insert(inputFile, blocks);
- }
- }
-
- @VisibleForTesting
- int getBlockSize(String languageKey) {
- int blockSize = settings.getInt("sonar.cpd." + languageKey + ".minimumLines");
- if (blockSize == 0) {
- blockSize = getDefaultBlockSize(languageKey);
- }
- return blockSize;
- }
-
- @VisibleForTesting
- static int getDefaultBlockSize(String languageKey) {
- if ("cobol".equals(languageKey)) {
- return 30;
- } else if ("abap".equals(languageKey)) {
- return 20;
- } else {
- return 10;
- }
- }
-
- @VisibleForTesting
- int getMinimumTokens(String languageKey) {
- int minimumTokens = settings.getInt("sonar.cpd." + languageKey + ".minimumTokens");
- if (minimumTokens == 0) {
- minimumTokens = 100;
- }
-
- return minimumTokens;
- }
-
-}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.batch.cpd;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.CpdMapping;
+import org.sonar.api.batch.fs.FilePredicates;
+import org.sonar.api.batch.fs.FileSystem;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.config.Settings;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.duplications.block.Block;
+import org.sonar.duplications.internal.pmd.TokenizerBridge;
+
+public class DefaultCpdIndexer extends CpdIndexer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultCpdIndexer.class);
+
+ private final CpdMappings mappings;
+ private final FileSystem fs;
+ private final Settings settings;
+ private final SonarDuplicationsIndex index;
+
+ public DefaultCpdIndexer(CpdMappings mappings, FileSystem fs, Settings settings, SonarDuplicationsIndex index) {
+ this.mappings = mappings;
+ this.fs = fs;
+ this.settings = settings;
+ this.index = index;
+ }
+
+ @Override
+ public boolean isLanguageSupported(String language) {
+ return true;
+ }
+
+ @Override
+ public void index(String languageKey) {
+ CpdMapping mapping = mappings.getMapping(languageKey);
+ if (mapping == null) {
+ LOG.debug("No CpdMapping for language " + languageKey);
+ return;
+ }
+
+ String[] cpdExclusions = settings.getStringArray(CoreProperties.CPD_EXCLUSIONS);
+ logExclusions(cpdExclusions, LOG);
+ FilePredicates p = fs.predicates();
+ List<InputFile> sourceFiles = Lists.newArrayList(fs.inputFiles(p.and(
+ p.hasType(InputFile.Type.MAIN),
+ p.hasLanguage(languageKey),
+ p.doesNotMatchPathPatterns(cpdExclusions))));
+ if (sourceFiles.isEmpty()) {
+ return;
+ }
+
+ // Create index
+ populateIndex(languageKey, sourceFiles, mapping, index);
+ }
+
+ private void populateIndex(String languageKey, List<InputFile> sourceFiles, CpdMapping mapping, SonarDuplicationsIndex index) {
+ TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), fs.encoding().name(), getBlockSize(languageKey));
+ for (InputFile inputFile : sourceFiles) {
+ LOG.debug("Populating index from {}", inputFile);
+ String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
+ List<Block> blocks = bridge.chunk(resourceEffectiveKey, inputFile.file());
+ index.insert(inputFile, blocks);
+ }
+ }
+
+ @VisibleForTesting
+ int getBlockSize(String languageKey) {
+ int blockSize = settings.getInt("sonar.cpd." + languageKey + ".minimumLines");
+ if (blockSize == 0) {
+ blockSize = getDefaultBlockSize(languageKey);
+ }
+ return blockSize;
+ }
+
+ @VisibleForTesting
+ static int getDefaultBlockSize(String languageKey) {
+ if ("cobol".equals(languageKey)) {
+ return 30;
+ } else if ("abap".equals(languageKey)) {
+ return 20;
+ } else {
+ return 10;
+ }
+ }
+
+}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-package org.sonar.batch.cpd;
-
-import com.google.common.collect.Lists;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.FilePredicates;
-import org.sonar.api.batch.fs.FileSystem;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.config.Settings;
-import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
-import org.sonar.batch.index.BatchComponentCache;
-import org.sonar.batch.report.ReportPublisher;
-import org.sonar.duplications.block.Block;
-import org.sonar.duplications.block.BlockChunker;
-import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm;
-import org.sonar.duplications.index.CloneGroup;
-import org.sonar.duplications.index.CloneIndex;
-import org.sonar.duplications.java.JavaStatementBuilder;
-import org.sonar.duplications.java.JavaTokenProducer;
-import org.sonar.duplications.statement.Statement;
-import org.sonar.duplications.statement.StatementChunker;
-import org.sonar.duplications.token.TokenChunker;
-
-public class JavaCpdEngine extends AbstractCpdEngine {
-
- private static final Logger LOG = LoggerFactory.getLogger(JavaCpdEngine.class);
-
- private static final int BLOCK_SIZE = 10;
-
- /**
- * Limit of time to analyse one file (in seconds).
- */
- private static final int TIMEOUT = 5 * 60;
-
- private final FileSystem fs;
- private final Settings settings;
- private final ReportPublisher publisher;
- private final BatchComponentCache batchComponentCache;
-
- public JavaCpdEngine(FileSystem fs, Settings settings, ReportPublisher publisher, BatchComponentCache batchComponentCache) {
- super(publisher, batchComponentCache);
- this.fs = fs;
- this.settings = settings;
- this.publisher = publisher;
- this.batchComponentCache = batchComponentCache;
- }
-
- @Override
- public boolean isLanguageSupported(String language) {
- return "java".equals(language);
- }
-
- @Override
- public void analyse(String languageKey, SensorContext context) {
- String[] cpdExclusions = settings.getStringArray(CoreProperties.CPD_EXCLUSIONS);
- logExclusions(cpdExclusions, LOG);
- FilePredicates p = fs.predicates();
- List<InputFile> sourceFiles = Lists.newArrayList(fs.inputFiles(p.and(
- p.hasType(InputFile.Type.MAIN),
- p.hasLanguage(languageKey),
- p.doesNotMatchPathPatterns(cpdExclusions))));
- if (sourceFiles.isEmpty()) {
- return;
- }
- SonarDuplicationsIndex index = createIndex(sourceFiles);
- detect(index, context, sourceFiles);
- }
-
- private SonarDuplicationsIndex createIndex(Iterable<InputFile> sourceFiles) {
- final SonarDuplicationsIndex index = new SonarDuplicationsIndex(publisher, batchComponentCache, settings);
-
- TokenChunker tokenChunker = JavaTokenProducer.build();
- StatementChunker statementChunker = JavaStatementBuilder.build();
- BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE);
-
- for (InputFile inputFile : sourceFiles) {
- LOG.debug("Populating index from {}", inputFile);
- String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
-
- List<Statement> statements;
-
- Reader reader = null;
- try {
- reader = new InputStreamReader(new FileInputStream(inputFile.file()), fs.encoding());
- statements = statementChunker.chunk(tokenChunker.chunk(reader));
- } catch (FileNotFoundException e) {
- throw new IllegalStateException("Cannot find file " + inputFile.file(), e);
- } finally {
- IOUtils.closeQuietly(reader);
- }
-
- List<Block> blocks = blockChunker.chunk(resourceEffectiveKey, statements);
- index.insert(inputFile, blocks);
- }
-
- return index;
- }
-
- private void detect(SonarDuplicationsIndex index, org.sonar.api.batch.sensor.SensorContext context, List<InputFile> sourceFiles) {
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- try {
- for (InputFile inputFile : sourceFiles) {
- LOG.debug("Detection of duplications for {}", inputFile);
- String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
-
- Collection<Block> fileBlocks = index.getByInputFile(inputFile, resourceEffectiveKey);
-
- List<CloneGroup> clones;
- try {
- clones = executorService.submit(new Task(index, fileBlocks)).get(TIMEOUT, TimeUnit.SECONDS);
- } catch (TimeoutException e) {
- clones = Collections.emptyList();
- LOG.warn("Timeout during detection of duplications for " + inputFile, e);
- } catch (InterruptedException | ExecutionException e) {
- throw new IllegalStateException("Fail during detection of duplication for " + inputFile, e);
- }
-
- saveDuplications(inputFile, clones);
- }
- } finally {
- executorService.shutdown();
- }
- }
-
- static class Task implements Callable<List<CloneGroup>> {
- private final CloneIndex index;
- private final Collection<Block> fileBlocks;
-
- public Task(CloneIndex index, Collection<Block> fileBlocks) {
- this.index = index;
- this.fileBlocks = fileBlocks;
- }
-
- @Override
- public List<CloneGroup> call() {
- return SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks);
- }
- }
-
-}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.batch.cpd;
+
+import com.google.common.collect.Lists;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.List;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.fs.FilePredicates;
+import org.sonar.api.batch.fs.FileSystem;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.config.Settings;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.duplications.block.Block;
+import org.sonar.duplications.block.BlockChunker;
+import org.sonar.duplications.java.JavaStatementBuilder;
+import org.sonar.duplications.java.JavaTokenProducer;
+import org.sonar.duplications.statement.Statement;
+import org.sonar.duplications.statement.StatementChunker;
+import org.sonar.duplications.token.TokenChunker;
+
+public class JavaCpdIndexer extends CpdIndexer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JavaCpdIndexer.class);
+
+ private static final int BLOCK_SIZE = 10;
+
+ private final FileSystem fs;
+ private final Settings settings;
+ private final SonarDuplicationsIndex index;
+
+ public JavaCpdIndexer(FileSystem fs, Settings settings, SonarDuplicationsIndex index) {
+ this.fs = fs;
+ this.settings = settings;
+ this.index = index;
+ }
+
+ @Override
+ public boolean isLanguageSupported(String language) {
+ return "java".equals(language);
+ }
+
+ @Override
+ public void index(String languageKey) {
+ String[] cpdExclusions = settings.getStringArray(CoreProperties.CPD_EXCLUSIONS);
+ logExclusions(cpdExclusions, LOG);
+ FilePredicates p = fs.predicates();
+ List<InputFile> sourceFiles = Lists.newArrayList(fs.inputFiles(p.and(
+ p.hasType(InputFile.Type.MAIN),
+ p.hasLanguage(languageKey),
+ p.doesNotMatchPathPatterns(cpdExclusions))));
+ if (sourceFiles.isEmpty()) {
+ return;
+ }
+ createIndex(sourceFiles);
+ }
+
+ private void createIndex(Iterable<InputFile> sourceFiles) {
+ TokenChunker tokenChunker = JavaTokenProducer.build();
+ StatementChunker statementChunker = JavaStatementBuilder.build();
+ BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE);
+
+ for (InputFile inputFile : sourceFiles) {
+ LOG.debug("Populating index from {}", inputFile);
+ String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
+
+ List<Statement> statements;
+
+ Reader reader = null;
+ try {
+ reader = new InputStreamReader(new FileInputStream(inputFile.file()), fs.encoding());
+ statements = statementChunker.chunk(tokenChunker.chunk(reader));
+ } catch (FileNotFoundException e) {
+ throw new IllegalStateException("Cannot find file " + inputFile.file(), e);
+ } finally {
+ IOUtils.closeQuietly(reader);
+ }
+
+ List<Block> blocks = blockChunker.chunk(resourceEffectiveKey, statements);
+ index.insert(inputFile, blocks);
+ }
+ }
+}
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import java.util.Collection;
+import java.util.Iterator;
+
import org.apache.commons.lang.StringUtils;
import org.sonar.api.CoreProperties;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.duplications.index.AbstractCloneIndex;
import org.sonar.duplications.index.CloneIndex;
import org.sonar.duplications.index.PackedMemoryCloneIndex;
+import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks;
public class SonarDuplicationsIndex extends AbstractCloneIndex {
&& StringUtils.isBlank(settings.getString(CoreProperties.PROJECT_BRANCH_PROPERTY));
}
- public Collection<Block> getByInputFile(InputFile inputFile, String resourceKey) {
+ public Collection<Block> getByInputFile(String resourceKey) {
return mem.getByResourceId(resourceKey);
}
throw new UnsupportedOperationException();
}
+ @Override
+ public Iterator<ResourceBlocks> iterator() {
+ return mem.iterator();
+ }
+
}
import org.sonar.api.batch.SensorContext;
import org.sonar.api.resources.Project;
import org.sonar.batch.analysis.DefaultAnalysisMode;
+import org.sonar.batch.cpd.CpdExecutor;
import org.sonar.batch.events.BatchStepEvent;
import org.sonar.batch.events.EventBus;
import org.sonar.batch.index.DefaultIndex;
private final DefaultAnalysisMode analysisMode;
private final IssueTransition localIssueTracking;
private final IssueCallback issueCallback;
+ private final CpdExecutor cpdExecutor;
public PhaseExecutor(InitializersExecutor initializersExecutor, PostJobsExecutor postJobsExecutor, SensorsExecutor sensorsExecutor,
SensorContext sensorContext, DefaultIndex index,
EventBus eventBus, ReportPublisher reportPublisher, ProjectInitializer pi,
FileSystemLogger fsLogger, IssuesReports jsonReport, DefaultModuleFileSystem fs, QProfileVerifier profileVerifier,
- IssueExclusionsLoader issueExclusionsLoader, DefaultAnalysisMode analysisMode, IssueTransition localIssueTracking, IssueCallback issueCallback) {
+ IssueExclusionsLoader issueExclusionsLoader, DefaultAnalysisMode analysisMode, IssueTransition localIssueTracking, IssueCallback issueCallback,
+ CpdExecutor cpdExecutor) {
this.postJobsExecutor = postJobsExecutor;
this.initializersExecutor = initializersExecutor;
this.sensorsExecutor = sensorsExecutor;
this.analysisMode = analysisMode;
this.localIssueTracking = localIssueTracking;
this.issueCallback = issueCallback;
+ this.cpdExecutor = cpdExecutor;
}
/**
if (analysisMode.isIssues()) {
localIssueTracking();
issuesCallback();
+ } else {
+ computeDuplications();
}
issuesReport();
publishReportJob();
eventBus.fireEvent(new ProjectAnalysisEvent(module, false));
}
+ private void computeDuplications() {
+ String stepName = "Computing duplications";
+ eventBus.fireEvent(new BatchStepEvent(stepName, true));
+ cpdExecutor.execute();
+ eventBus.fireEvent(new BatchStepEvent(stepName, false));
+ }
+
private void publishReportJob() {
String stepName = "Publish report";
eventBus.fireEvent(new BatchStepEvent(stepName, true));
import org.sonar.batch.bootstrap.ExtensionUtils;
import org.sonar.batch.bootstrap.MetricProvider;
import org.sonar.batch.cache.ProjectPersistentCacheProvider;
+import org.sonar.batch.cpd.CpdExecutor;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
import org.sonar.batch.events.EventBus;
import org.sonar.batch.index.BatchComponentCache;
import org.sonar.batch.index.Caches;
CoveragePublisher.class,
SourcePublisher.class,
TestExecutionAndCoveragePublisher.class,
+
+ // Cpd
+ CpdExecutor.class,
+ SonarDuplicationsIndex.class,
ScanTaskObservers.class,
UserRepositoryLoader.class);
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.batch.cpd;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.resources.Project;
-import org.sonar.api.utils.log.LogTester;
-import org.sonar.api.utils.log.LoggerLevel;
-import org.sonar.batch.index.BatchComponent;
-import org.sonar.batch.index.BatchComponentCache;
-import org.sonar.batch.protocol.output.BatchReport.Duplication;
-import org.sonar.batch.protocol.output.BatchReportReader;
-import org.sonar.batch.protocol.output.BatchReportWriter;
-import org.sonar.batch.report.ReportPublisher;
-import org.sonar.core.util.CloseableIterator;
-import org.sonar.duplications.index.CloneGroup;
-import org.sonar.duplications.index.ClonePart;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class AbstractCpdEngineTest {
-
- @Rule
- public LogTester logTester = new LogTester();
-
- @Rule
- public TemporaryFolder temp = new TemporaryFolder();
-
- private AbstractCpdEngine engine;
-
- private BatchReportReader reader;
- private DefaultInputFile inputFile1;
- private BatchComponent batchComponent1;
- private BatchComponent batchComponent2;
- private BatchComponent batchComponent3;
-
- @Before
- public void before() throws IOException {
- File outputDir = temp.newFolder();
- ReportPublisher reportPublisher = mock(ReportPublisher.class);
- when(reportPublisher.getWriter()).thenReturn(new BatchReportWriter(outputDir));
- reader = new BatchReportReader(outputDir);
- BatchComponentCache componentCache = new BatchComponentCache();
- Project p = new Project("foo");
- componentCache.add(p, null).setInputComponent(new DefaultInputModule("foo"));
- org.sonar.api.resources.Resource sampleFile = org.sonar.api.resources.File.create("src/Foo.php").setEffectiveKey("foo:src/Foo.php");
- inputFile1 = new DefaultInputFile("foo", "src/Foo.php").setLines(5);
- batchComponent1 = componentCache.add(sampleFile, null).setInputComponent(inputFile1);
- org.sonar.api.resources.Resource sampleFile2 = org.sonar.api.resources.File.create("src/Foo2.php").setEffectiveKey("foo:src/Foo2.php");
- batchComponent2 = componentCache.add(sampleFile2, null).setInputComponent(new DefaultInputFile("foo", "src/Foo2.php").setLines(5));
- org.sonar.api.resources.Resource sampleFile3 = org.sonar.api.resources.File.create("src/Foo3.php").setEffectiveKey("foo:src/Foo3.php");
- batchComponent3 = componentCache.add(sampleFile3, null).setInputComponent(new DefaultInputFile("foo", "src/Foo3.php").setLines(5));
- engine = new AbstractCpdEngine(reportPublisher, componentCache) {
-
- @Override
- boolean isLanguageSupported(String language) {
- return false;
- }
-
- @Override
- void analyse(String language, SensorContext context) {
- }
- };
- }
-
- @Test
- public void testNothingToSave() {
- engine.saveDuplications(inputFile1, Collections.EMPTY_LIST);
-
- assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(0);
- }
-
- @Test
- public void testOneSimpleDuplicationBetweenTwoFiles() {
- List<CloneGroup> groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 2, 4), new ClonePart(batchComponent2.key(), 0, 15, 17)));
- engine.saveDuplications(inputFile1, groups);
-
- assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1);
- CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
- Duplication duplication = dups.next();
- dups.close();
- assertThat(duplication.getOriginPosition().getStartLine()).isEqualTo(2);
- assertThat(duplication.getOriginPosition().getEndLine()).isEqualTo(4);
- assertThat(duplication.getDuplicateList()).hasSize(1);
- assertThat(duplication.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent2.batchId());
- assertThat(duplication.getDuplicate(0).getRange().getStartLine()).isEqualTo(15);
- assertThat(duplication.getDuplicate(0).getRange().getEndLine()).isEqualTo(17);
- }
-
- @Test
- public void testDuplicationOnSameFile() throws Exception {
- List<CloneGroup> groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent1.key(), 0, 215, 414)));
- engine.saveDuplications(inputFile1, groups);
-
- assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1);
- CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
- Duplication duplication = dups.next();
- dups.close();
- assertThat(duplication.getOriginPosition().getStartLine()).isEqualTo(5);
- assertThat(duplication.getOriginPosition().getEndLine()).isEqualTo(204);
- assertThat(duplication.getDuplicateList()).hasSize(1);
- assertThat(duplication.getDuplicate(0).hasOtherFileRef()).isFalse();
- assertThat(duplication.getDuplicate(0).getRange().getStartLine()).isEqualTo(215);
- assertThat(duplication.getDuplicate(0).getRange().getEndLine()).isEqualTo(414);
- }
-
- @Test
- public void testTooManyDuplicates() throws Exception {
- // 1 origin part + 101 duplicates = 102
- List<ClonePart> parts = new ArrayList<>(AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP + 2);
- for (int i = 0; i < AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP + 2; i++) {
- parts.add(new ClonePart(batchComponent1.key(), i, i, i + 1));
- }
- List<CloneGroup> groups = Arrays.asList(CloneGroup.builder().setLength(0).setOrigin(parts.get(0)).setParts(parts).build());
- engine.saveDuplications(inputFile1, groups);
-
- assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1);
- CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
- Duplication duplication = dups.next();
- dups.close();
- assertThat(duplication.getDuplicateList()).hasSize(AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP);
-
- assertThat(logTester.logs(LoggerLevel.WARN)).contains("Too many duplication references on file " + inputFile1.relativePath() + " for block at line 0. Keep only the first "
- + AbstractCpdEngine.MAX_CLONE_PART_PER_GROUP + " references.");
- }
-
- @Test
- public void testTooManyDuplications() throws Exception {
- // 1 origin part + 101 duplicates = 102
- List<CloneGroup> dups = new ArrayList<>(AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE + 1);
- for (int i = 0; i < AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE + 1; i++) {
- ClonePart clonePart = new ClonePart(batchComponent1.key(), i, i, i + 1);
- ClonePart dupPart = new ClonePart(batchComponent1.key(), i + 1, i + 1, i + 2);
- dups.add(newCloneGroup(clonePart, dupPart));
- }
- engine.saveDuplications(inputFile1, dups);
-
- assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE);
-
- assertThat(logTester.logs(LoggerLevel.WARN))
- .contains("Too many duplication groups on file " + inputFile1.relativePath() + ". Keep only the first " + AbstractCpdEngine.MAX_CLONE_GROUP_PER_FILE + " groups.");
- }
-
- @Test
- public void testOneDuplicatedGroupInvolvingMoreThanTwoFiles() throws Exception {
- List<CloneGroup> groups = Arrays
- .asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 25, 224)));
- engine.saveDuplications(inputFile1, groups);
-
- assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(1);
- CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
- Duplication duplication = dups.next();
- dups.close();
- assertThat(duplication.getOriginPosition().getStartLine()).isEqualTo(5);
- assertThat(duplication.getOriginPosition().getEndLine()).isEqualTo(204);
- assertThat(duplication.getDuplicateList()).hasSize(2);
- assertThat(duplication.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent2.batchId());
- assertThat(duplication.getDuplicate(0).getRange().getStartLine()).isEqualTo(15);
- assertThat(duplication.getDuplicate(0).getRange().getEndLine()).isEqualTo(214);
- assertThat(duplication.getDuplicate(1).getOtherFileRef()).isEqualTo(batchComponent3.batchId());
- assertThat(duplication.getDuplicate(1).getRange().getStartLine()).isEqualTo(25);
- assertThat(duplication.getDuplicate(1).getRange().getEndLine()).isEqualTo(224);
- }
-
- @Test
- public void testTwoDuplicatedGroupsInvolvingThreeFiles() throws Exception {
- List<CloneGroup> groups = Arrays.asList(
- newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214)),
- newCloneGroup(new ClonePart(batchComponent1.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 15, 214)));
- engine.saveDuplications(inputFile1, groups);
-
- assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(2);
- CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
- Duplication duplication1 = dups.next();
- Duplication duplication2 = dups.next();
- dups.close();
- assertThat(duplication1.getOriginPosition().getStartLine()).isEqualTo(5);
- assertThat(duplication1.getOriginPosition().getEndLine()).isEqualTo(204);
- assertThat(duplication1.getDuplicateList()).hasSize(1);
- assertThat(duplication1.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent2.batchId());
- assertThat(duplication1.getDuplicate(0).getRange().getStartLine()).isEqualTo(15);
- assertThat(duplication1.getDuplicate(0).getRange().getEndLine()).isEqualTo(214);
-
- assertThat(duplication2.getOriginPosition().getStartLine()).isEqualTo(15);
- assertThat(duplication2.getOriginPosition().getEndLine()).isEqualTo(214);
- assertThat(duplication2.getDuplicateList()).hasSize(1);
- assertThat(duplication2.getDuplicate(0).getOtherFileRef()).isEqualTo(batchComponent3.batchId());
- assertThat(duplication2.getDuplicate(0).getRange().getStartLine()).isEqualTo(15);
- assertThat(duplication2.getDuplicate(0).getRange().getEndLine()).isEqualTo(214);
- }
-
- private CloneGroup newCloneGroup(ClonePart... parts) {
- return CloneGroup.builder().setLength(0).setOrigin(parts[0]).setParts(Arrays.asList(parts)).build();
- }
-
-}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.batch.cpd;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.api.config.Settings;
+import org.sonar.api.resources.Project;
+import org.sonar.api.utils.log.LogTester;
+import org.sonar.api.utils.log.LoggerLevel;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.index.BatchComponent;
+import org.sonar.batch.index.BatchComponentCache;
+import org.sonar.batch.protocol.output.BatchReportReader;
+import org.sonar.batch.protocol.output.BatchReportWriter;
+import org.sonar.batch.protocol.output.BatchReport.Duplicate;
+import org.sonar.batch.protocol.output.BatchReport.Duplication;
+import org.sonar.batch.report.ReportPublisher;
+import org.sonar.core.util.CloseableIterator;
+import org.sonar.duplications.index.CloneGroup;
+import org.sonar.duplications.index.ClonePart;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CpdExecutorTest {
+ private CpdExecutor executor;
+ private Settings settings;
+ private SonarDuplicationsIndex index;
+ private ReportPublisher publisher;
+ private BatchComponentCache componentCache;
+
+ @Rule
+ public LogTester logTester = new LogTester();
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ // private AbstractCpdEngine engine;
+
+ private BatchReportReader reader;
+ private BatchComponent batchComponent1;
+ private BatchComponent batchComponent2;
+ private BatchComponent batchComponent3;
+
+ @Before
+ public void setUp() throws IOException {
+ File outputDir = temp.newFolder();
+
+ settings = new Settings();
+ index = mock(SonarDuplicationsIndex.class);
+ publisher = mock(ReportPublisher.class);
+ when(publisher.getWriter()).thenReturn(new BatchReportWriter(outputDir));
+ componentCache = new BatchComponentCache();
+ executor = new CpdExecutor(settings, index, publisher, componentCache);
+ reader = new BatchReportReader(outputDir);
+
+ Project p = new Project("foo");
+ componentCache.add(p, null).setInputComponent(new DefaultInputModule("foo"));
+
+ batchComponent1 = createComponent("src/Foo.php", 5);
+ batchComponent2 = createComponent("src/Foo2.php", 5);
+ batchComponent3 = createComponent("src/Foo3.php", 5);
+ }
+
+ private BatchComponent createComponent(String relativePath, int lines) {
+ org.sonar.api.resources.Resource sampleFile = org.sonar.api.resources.File.create("relativePath").setEffectiveKey("foo:" + relativePath);
+ return componentCache.add(sampleFile, null).setInputComponent(new DefaultInputFile("foo", relativePath).setLines(lines));
+ }
+
+ @Test
+ public void defaultMinimumTokens() {
+ assertThat(executor.getMinimumTokens("java")).isEqualTo(100);
+ }
+
+ @Test
+ public void minimumTokensByLanguage() {
+ settings.setProperty("sonar.cpd.java.minimumTokens", "42");
+ settings.setProperty("sonar.cpd.php.minimumTokens", "33");
+ assertThat(executor.getMinimumTokens("java")).isEqualTo(42);
+
+ settings.setProperty("sonar.cpd.java.minimumTokens", "42");
+ settings.setProperty("sonar.cpd.php.minimumTokens", "33");
+ assertThat(executor.getMinimumTokens("php")).isEqualTo(33);
+ }
+
+ @Test
+ public void testNothingToSave() {
+ executor.saveDuplications(batchComponent1, Collections.<CloneGroup>emptyList());
+ assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(0);
+ }
+
+ @Test
+ public void reportOneSimpleDuplicationBetweenTwoFiles() {
+ List<CloneGroup> groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 2, 4), new ClonePart(batchComponent2.key(), 0, 15, 17)));
+
+ executor.saveDuplications(batchComponent1, groups);
+
+ Duplication[] dups = readDuplications(1);
+ assertDuplication(dups[0], 2, 4, batchComponent2.batchId(), 15, 17);
+ }
+
+ @Test
+ public void reportDuplicationOnSameFile() throws Exception {
+ List<CloneGroup> groups = Arrays.asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent1.key(), 0, 215, 414)));
+ executor.saveDuplications(batchComponent1, groups);
+
+ Duplication[] dups = readDuplications(1);
+ assertDuplication(dups[0], 5, 204, null, 215, 414);
+ }
+
+ @Test
+ public void reportTooManyDuplicates() throws Exception {
+ // 1 origin part + 101 duplicates = 102
+ List<ClonePart> parts = new ArrayList<>(CpdExecutor.MAX_CLONE_PART_PER_GROUP + 2);
+ for (int i = 0; i < CpdExecutor.MAX_CLONE_PART_PER_GROUP + 2; i++) {
+ parts.add(new ClonePart(batchComponent1.key(), i, i, i + 1));
+ }
+ List<CloneGroup> groups = Arrays.asList(CloneGroup.builder().setLength(0).setOrigin(parts.get(0)).setParts(parts).build());
+ executor.saveDuplications(batchComponent1, groups);
+
+ Duplication[] dups = readDuplications(1);
+ assertThat(dups[0].getDuplicateList()).hasSize(CpdExecutor.MAX_CLONE_PART_PER_GROUP);
+
+ assertThat(logTester.logs(LoggerLevel.WARN))
+ .contains("Too many duplication references on file " + batchComponent1.inputComponent() + " for block at line 0. Keep only the first "
+ + CpdExecutor.MAX_CLONE_PART_PER_GROUP + " references.");
+ }
+
+ @Test
+ public void reportTooManyDuplications() throws Exception {
+ // 1 origin part + 101 duplicates = 102
+ List<CloneGroup> dups = new ArrayList<>(CpdExecutor.MAX_CLONE_GROUP_PER_FILE + 1);
+ for (int i = 0; i < CpdExecutor.MAX_CLONE_GROUP_PER_FILE + 1; i++) {
+ ClonePart clonePart = new ClonePart(batchComponent1.key(), i, i, i + 1);
+ ClonePart dupPart = new ClonePart(batchComponent1.key(), i + 1, i + 1, i + 2);
+ dups.add(newCloneGroup(clonePart, dupPart));
+ }
+ executor.saveDuplications(batchComponent1, dups);
+
+ assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(CpdExecutor.MAX_CLONE_GROUP_PER_FILE);
+
+ assertThat(logTester.logs(LoggerLevel.WARN))
+ .contains("Too many duplication groups on file " + batchComponent1.inputComponent() + ". Keep only the first " + CpdExecutor.MAX_CLONE_GROUP_PER_FILE + " groups.");
+ }
+
+ @Test
+ public void reportOneDuplicatedGroupInvolvingMoreThanTwoFiles() throws Exception {
+ List<CloneGroup> groups = Arrays
+ .asList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 25, 224)));
+ executor.saveDuplications(batchComponent1, groups);
+
+ Duplication[] dups = readDuplications(1);
+ assertDuplication(dups[0], 5, 204, 2);
+ assertDuplicate(dups[0].getDuplicate(0), batchComponent2.batchId(), 15, 214);
+ assertDuplicate(dups[0].getDuplicate(1), batchComponent3.batchId(), 25, 224);
+ }
+
+ @Test
+ public void reportTwoDuplicatedGroupsInvolvingThreeFiles() throws Exception {
+ List<CloneGroup> groups = Arrays.asList(
+ newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent2.key(), 0, 15, 214)),
+ newCloneGroup(new ClonePart(batchComponent1.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 15, 214)));
+ executor.saveDuplications(batchComponent1, groups);
+
+ Duplication[] dups = readDuplications(2);
+ assertDuplication(dups[0], 5, 204, batchComponent2.batchId(), 15, 214);
+ assertDuplication(dups[1], 15, 214, batchComponent3.batchId(), 15, 214);
+ }
+
+ private Duplication[] readDuplications(int expected) {
+ assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(expected);
+ Duplication[] duplications = new Duplication[expected];
+ CloseableIterator<Duplication> dups = reader.readComponentDuplications(batchComponent1.batchId());
+
+ for(int i = 0; i< expected; i++) {
+ duplications[i] = dups.next();
+ }
+ dups.close();
+ return duplications;
+ }
+
+ private void assertDuplicate(Duplicate d, int otherFileRef, int rangeStartLine, int rangeEndLine) {
+ assertThat(d.getOtherFileRef()).isEqualTo(otherFileRef);
+ assertThat(d.getRange().getStartLine()).isEqualTo(rangeStartLine);
+ assertThat(d.getRange().getEndLine()).isEqualTo(rangeEndLine);
+ }
+
+ private void assertDuplication(Duplication d, int originStartLine, int originEndLine, int numDuplicates) {
+ assertThat(d.getOriginPosition().getStartLine()).isEqualTo(originStartLine);
+ assertThat(d.getOriginPosition().getEndLine()).isEqualTo(originEndLine);
+ assertThat(d.getDuplicateList()).hasSize(numDuplicates);
+ }
+
+ private void assertDuplication(Duplication d, int originStartLine, int originEndLine, Integer otherFileRef, int rangeStartLine, int rangeEndLine) {
+ assertThat(d.getOriginPosition().getStartLine()).isEqualTo(originStartLine);
+ assertThat(d.getOriginPosition().getEndLine()).isEqualTo(originEndLine);
+ assertThat(d.getDuplicateList()).hasSize(1);
+ if(otherFileRef != null) {
+ assertThat(d.getDuplicate(0).getOtherFileRef()).isEqualTo(otherFileRef);
+ } else {
+ assertThat(d.getDuplicate(0).hasOtherFileRef()).isFalse();
+ }
+ assertThat(d.getDuplicate(0).getRange().getStartLine()).isEqualTo(rangeStartLine);
+ assertThat(d.getDuplicate(0).getRange().getEndLine()).isEqualTo(rangeEndLine);
+ }
+
+ private CloneGroup newCloneGroup(ClonePart... parts) {
+ return CloneGroup.builder().setLength(0).setOrigin(parts[0]).setParts(Arrays.asList(parts)).build();
+ }
+}
@Rule
public TemporaryFolder temp = new TemporaryFolder();
- JavaCpdEngine sonarEngine;
- DefaultCpdEngine sonarBridgeEngine;
+ JavaCpdIndexer sonarEngine;
+ DefaultCpdIndexer sonarBridgeEngine;
CpdSensor sensor;
Settings settings;
@Before
public void setUp() throws IOException {
- sonarEngine = new JavaCpdEngine(null, null, null, null);
- sonarBridgeEngine = new DefaultCpdEngine(new CpdMappings(), null, null, null, null);
+ sonarEngine = new JavaCpdIndexer(null, null, null);
+ sonarBridgeEngine = new DefaultCpdIndexer(new CpdMappings(), null, null, null);
settings = new Settings(new PropertyDefinitions(CpdComponents.class));
DefaultFileSystem fs = new DefaultFileSystem(temp.newFolder().toPath());
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.batch.cpd;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.sonar.api.config.Settings;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-public class DefaultCpdEngineTest {
-
- private DefaultCpdEngine engine;
- private Settings settings;
-
- @Before
- public void init() {
- settings = new Settings();
- engine = new DefaultCpdEngine(null, null, settings, null, null);
- }
-
- @Test
- public void shouldLogExclusions() {
- Logger logger = mock(Logger.class);
- engine.logExclusions(new String[0], logger);
- verify(logger, never()).info(anyString());
-
- logger = mock(Logger.class);
- engine.logExclusions(new String[] {"Foo*", "**/Bar*"}, logger);
-
- String message = "Copy-paste detection exclusions:"
- + "\n Foo*"
- + "\n **/Bar*";
- verify(logger, times(1)).info(message);
- }
-
- @Test
- public void shouldReturnDefaultBlockSize() {
- assertThat(DefaultCpdEngine.getDefaultBlockSize("cobol")).isEqualTo(30);
- assertThat(DefaultCpdEngine.getDefaultBlockSize("abap")).isEqualTo(20);
- assertThat(DefaultCpdEngine.getDefaultBlockSize("other")).isEqualTo(10);
- }
-
- @Test
- public void defaultBlockSize() {
-
- assertThat(engine.getBlockSize("java")).isEqualTo(10);
- }
-
- @Test
- public void blockSizeForCobol() {
- settings.setProperty("sonar.cpd.cobol.minimumLines", "42");
-
- assertThat(engine.getBlockSize("cobol")).isEqualTo(42);
- }
-
- @Test
- public void defaultMinimumTokens() {
- assertThat(engine.getMinimumTokens("java")).isEqualTo(100);
- }
-
- @Test
- public void minimumTokensByLanguage() {
- settings.setProperty("sonar.cpd.java.minimumTokens", "42");
- settings.setProperty("sonar.cpd.php.minimumTokens", "33");
- assertThat(engine.getMinimumTokens("java")).isEqualTo(42);
-
- settings.setProperty("sonar.cpd.java.minimumTokens", "42");
- settings.setProperty("sonar.cpd.php.minimumTokens", "33");
- assertThat(engine.getMinimumTokens("php")).isEqualTo(33);
- }
-
-}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.batch.cpd;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.sonar.api.config.Settings;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class DefaultCpdIndexerTest {
+
+ private DefaultCpdIndexer engine;
+ private Settings settings;
+
+ @Before
+ public void init() {
+ settings = new Settings();
+ engine = new DefaultCpdIndexer(null, null, settings, null);
+ }
+
+ @Test
+ public void shouldLogExclusions() {
+ Logger logger = mock(Logger.class);
+ engine.logExclusions(new String[0], logger);
+ verify(logger, never()).info(anyString());
+
+ logger = mock(Logger.class);
+ engine.logExclusions(new String[] {"Foo*", "**/Bar*"}, logger);
+
+ String message = "Copy-paste detection exclusions:"
+ + "\n Foo*"
+ + "\n **/Bar*";
+ verify(logger, times(1)).info(message);
+ }
+
+ @Test
+ public void shouldReturnDefaultBlockSize() {
+ assertThat(DefaultCpdIndexer.getDefaultBlockSize("cobol")).isEqualTo(30);
+ assertThat(DefaultCpdIndexer.getDefaultBlockSize("abap")).isEqualTo(20);
+ assertThat(DefaultCpdIndexer.getDefaultBlockSize("other")).isEqualTo(10);
+ }
+
+ @Test
+ public void defaultBlockSize() {
+
+ assertThat(engine.getBlockSize("java")).isEqualTo(10);
+ }
+
+ @Test
+ public void blockSizeForCobol() {
+ settings.setProperty("sonar.cpd.cobol.minimumLines", "42");
+
+ assertThat(engine.getBlockSize("cobol")).isEqualTo(42);
+ }
+
+}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.batch.cpd;
-
-import java.io.File;
-import org.apache.commons.io.FileUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.sonar.api.batch.fs.internal.DefaultFileSystem;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.config.Settings;
-import org.sonar.batch.index.BatchComponentCache;
-import org.sonar.batch.protocol.output.BatchReport;
-import org.sonar.batch.protocol.output.BatchReport.Duplication;
-import org.sonar.batch.protocol.output.BatchReportReader;
-import org.sonar.batch.protocol.output.BatchReportWriter;
-import org.sonar.batch.report.ReportPublisher;
-import org.sonar.core.util.CloseableIterator;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class JavaCpdEngineTest {
-
- private static final String JAVA = "java";
- @Rule
- public TemporaryFolder temp = new TemporaryFolder();
-
- @Test
- public void testJavaCodeWithTwoCloneGroupAtSameLines() throws Exception {
-
- File baseDir = temp.newFolder();
- DefaultFileSystem fs = new DefaultFileSystem(baseDir);
- DefaultInputFile file = new DefaultInputFile("foo", "src/ManyStatements.java").setLanguage(JAVA);
- fs.add(file);
- BatchComponentCache batchComponentCache = new BatchComponentCache();
- batchComponentCache.add(org.sonar.api.resources.File.create("src/Foo.java").setEffectiveKey("foo:src/ManyStatements.java"), null).setInputComponent(file);
- File ioFile = file.file();
- FileUtils.copyURLToFile(this.getClass().getResource("ManyStatements.java"), ioFile);
-
- File reportOut = temp.newFolder();
- ReportPublisher reportPublisher = mock(ReportPublisher.class);
- when(reportPublisher.getWriter()).thenReturn(new BatchReportWriter(reportOut));
- JavaCpdEngine engine = new JavaCpdEngine(fs, new Settings(), reportPublisher, batchComponentCache);
- engine.analyse(JAVA, mock(SensorContext.class));
-
- BatchReportReader reader = new BatchReportReader(reportOut);
- try (CloseableIterator<BatchReport.Duplication> it = reader.readComponentDuplications(1)) {
- Duplication dupGroup1 = it.next();
- Duplication dupGroup2 = it.next();
-
- assertThat(dupGroup1.getOriginPosition().getStartLine()).isEqualTo(6);
- assertThat(dupGroup1.getOriginPosition().getEndLine()).isEqualTo(6);
- assertThat(dupGroup1.getDuplicateCount()).isEqualTo(1);
- assertThat(dupGroup1.getDuplicate(0).getRange().getStartLine()).isEqualTo(8);
- assertThat(dupGroup1.getDuplicate(0).getRange().getEndLine()).isEqualTo(8);
-
- assertThat(dupGroup2.getOriginPosition().getStartLine()).isEqualTo(6);
- assertThat(dupGroup2.getOriginPosition().getEndLine()).isEqualTo(6);
- assertThat(dupGroup2.getDuplicateCount()).isEqualTo(2);
- assertThat(dupGroup2.getDuplicate(0).getRange().getStartLine()).isEqualTo(7);
- assertThat(dupGroup2.getDuplicate(0).getRange().getEndLine()).isEqualTo(7);
- assertThat(dupGroup2.getDuplicate(1).getRange().getStartLine()).isEqualTo(8);
- assertThat(dupGroup2.getDuplicate(1).getRange().getEndLine()).isEqualTo(8);
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
-
- }
-
-}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.batch.cpd;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.fs.FileSystem;
+import org.sonar.api.batch.fs.internal.DefaultFileSystem;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.config.Settings;
+import org.sonar.batch.cpd.index.SonarDuplicationsIndex;
+import org.sonar.batch.index.BatchComponentCache;
+import org.sonar.duplications.block.Block;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+
+public class JavaCpdIndexerTest {
+ private static final String JAVA = "java";
+
+ @Mock
+ private SonarDuplicationsIndex index;
+
+ @Captor
+ private ArgumentCaptor<List<Block>> blockCaptor;
+
+ private Settings settings;
+ private JavaCpdIndexer engine;
+ private DefaultInputFile file;
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ @Before
+ public void setUp() throws IOException {
+ MockitoAnnotations.initMocks(this);
+
+ File baseDir = temp.newFolder();
+ DefaultFileSystem fs = new DefaultFileSystem(baseDir);
+ file = new DefaultInputFile("foo", "src/ManyStatements.java").setLanguage(JAVA);
+ fs.add(file);
+ BatchComponentCache batchComponentCache = new BatchComponentCache();
+ batchComponentCache.add(org.sonar.api.resources.File.create("src/Foo.java").setEffectiveKey("foo:src/ManyStatements.java"), null).setInputComponent(file);
+ File ioFile = file.file();
+ FileUtils.copyURLToFile(this.getClass().getResource("ManyStatements.java"), ioFile);
+
+ settings = new Settings();
+ engine = new JavaCpdIndexer(fs, settings, index);
+ }
+
+ @Test
+ public void languageSupported() {
+ JavaCpdIndexer engine = new JavaCpdIndexer(mock(FileSystem.class), new Settings(), index);
+ assertThat(engine.isLanguageSupported(JAVA)).isTrue();
+ assertThat(engine.isLanguageSupported("php")).isFalse();
+ }
+
+ @Test
+ public void testExclusions() {
+ settings.setProperty(CoreProperties.CPD_EXCLUSIONS, "**");
+ engine.index(JAVA);
+ verifyZeroInteractions(index);
+ }
+
+ @Test
+ public void testJavaIndexing() throws Exception {
+ engine.index(JAVA);
+
+ verify(index).insert(eq(file), blockCaptor.capture());
+ List<Block> blockList = blockCaptor.getValue();
+
+ assertThat(blockList).hasSize(26);
+ }
+}
tester.stop();
}
+ @Test
+ public void testCrossModuleDuplications() throws IOException {
+ builder.put("sonar.modules", "module1,module2")
+ .put("sonar.cpd.xoo.minimumTokens", "10")
+ .put("sonar.verbose", "true");
+
+ // module 1
+ builder.put("module1.sonar.projectKey", "module1");
+ builder.put("module1.sonar.projectName", "Module 1");
+ builder.put("module1.sonar.sources", ".");
+
+ // module2
+ builder.put("module2.sonar.projectKey", "module2");
+ builder.put("module2.sonar.projectName", "Module 2");
+ builder.put("module2.sonar.sources", ".");
+
+ File module1Dir = new File(baseDir, "module1");
+ File module2Dir = new File(baseDir, "module2");
+
+ module1Dir.mkdir();
+ module2Dir.mkdir();
+
+ String duplicatedStuff = "Sample xoo\ncontent\n"
+ + "foo\nbar\ntoto\ntiti\n"
+ + "foo\nbar\ntoto\ntiti\n"
+ + "bar\ntoto\ntiti\n"
+ + "foo\nbar\ntoto\ntiti";
+
+ // create duplicated file in both modules
+ File xooFile1 = new File(module1Dir, "sample1.xoo");
+ FileUtils.write(xooFile1, duplicatedStuff);
+
+ File xooFile2 = new File(module2Dir, "sample2.xoo");
+ FileUtils.write(xooFile2, duplicatedStuff);
+
+ TaskResult result = tester.newTask().properties(builder.build()).start();
+
+ assertThat(result.inputFiles()).hasSize(2);
+
+ InputFile inputFile1 = result.inputFile("sample1.xoo");
+ InputFile inputFile2 = result.inputFile("sample2.xoo");
+
+ // One clone group on each file
+ List<org.sonar.batch.protocol.output.BatchReport.Duplication> duplicationGroupsFile1 = result.duplicationsFor(inputFile1);
+ assertThat(duplicationGroupsFile1).hasSize(1);
+
+ org.sonar.batch.protocol.output.BatchReport.Duplication cloneGroupFile1 = duplicationGroupsFile1.get(0);
+ assertThat(cloneGroupFile1.getOriginPosition().getStartLine()).isEqualTo(1);
+ assertThat(cloneGroupFile1.getOriginPosition().getEndLine()).isEqualTo(17);
+ assertThat(cloneGroupFile1.getDuplicateList()).hasSize(1);
+ assertThat(cloneGroupFile1.getDuplicate(0).getOtherFileRef()).isEqualTo(result.getReportComponent(((DefaultInputFile) inputFile2).key()).getRef());
+
+ List<org.sonar.batch.protocol.output.BatchReport.Duplication> duplicationGroupsFile2 = result.duplicationsFor(inputFile2);
+ assertThat(duplicationGroupsFile2).hasSize(1);
+
+ org.sonar.batch.protocol.output.BatchReport.Duplication cloneGroupFile2 = duplicationGroupsFile2.get(0);
+ assertThat(cloneGroupFile2.getOriginPosition().getStartLine()).isEqualTo(1);
+ assertThat(cloneGroupFile2.getOriginPosition().getEndLine()).isEqualTo(17);
+ assertThat(cloneGroupFile2.getDuplicateList()).hasSize(1);
+ assertThat(cloneGroupFile2.getDuplicate(0).getOtherFileRef()).isEqualTo(result.getReportComponent(((DefaultInputFile) inputFile1).key()).getRef());
+
+ assertThat(result.duplicationBlocksFor(inputFile1)).isEmpty();
+ }
+
@Test
public void testCrossFileDuplications() throws IOException {
File srcDir = new File(baseDir, "src");
srcDir.mkdir();
- String duplicatedStuff = "Sample xoo\ncontent\nfoo\nbar\ntoto\ntiti\nfoo\nbar\ntoto\ntiti\nbar\ntoto\ntiti\nfoo\nbar\ntoto\ntiti";
+ String duplicatedStuff = "Sample xoo\ncontent\n"
+ + "foo\nbar\ntoto\ntiti\n"
+ + "foo\nbar\ntoto\ntiti\n"
+ + "bar\ntoto\ntiti\n"
+ + "foo\nbar\ntoto\ntiti";
File xooFile1 = new File(srcDir, "sample1.xoo");
FileUtils.write(xooFile1, duplicatedStuff);
assertThat(result.inputFiles()).hasSize(2);
- Map<String, List<org.sonar.batch.protocol.output.BatchReport.Measure>> allMeasures = result.allMeasures();
-
InputFile inputFile1 = result.inputFile("src/sample1.xoo");
InputFile inputFile2 = result.inputFile("src/sample2.xoo");
package org.sonar.duplications.index;
import java.util.Collection;
+import java.util.Iterator;
import org.sonar.duplications.block.Block;
import org.sonar.duplications.block.ByteArray;
+import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks;
public interface CloneIndex {
*/
void insert(Block block);
+ /**
+ * Iterators through the resources, providing the list of blocks for each resource.
+ */
+ Iterator<ResourceBlocks> iterator();
+
}
import com.google.common.collect.Multimap;
import org.sonar.duplications.block.Block;
import org.sonar.duplications.block.ByteArray;
+import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks;
import java.util.Collection;
+import java.util.Iterator;
public class MemoryCloneIndex implements CloneIndex {
byHash.put(block.getBlockHash(), block);
}
+ @Override
+ public Iterator<ResourceBlocks> iterator() {
+ throw new UnsupportedOperationException();
+ }
+
}
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Iterator;
import java.util.List;
+import java.util.NoSuchElementException;
+
import org.sonar.duplications.block.Block;
import org.sonar.duplications.block.ByteArray;
import org.sonar.duplications.utils.FastStringComparator;
+import javax.annotation.Nullable;
+
/**
* Provides an index optimized by memory.
* <p>
List<Block> result = new ArrayList<>();
int realIndex = resourceIdsIndex[index];
while (index < size && FastStringComparator.INSTANCE.compare(resourceIds[realIndex], resourceId) == 0) {
- // extract block (note that there is no need to extract resourceId)
- int offset = realIndex * blockInts;
+ result.add(getBlock(realIndex, resourceId));
+
+ index++;
+ realIndex = resourceIdsIndex[index];
+ }
+ return result;
+ }
+
+ private Block createBlock(int index, String resourceId, @Nullable ByteArray byteHash) {
+ int offset = index * blockInts;
+ ByteArray blockHash;
+
+ if (byteHash == null) {
int[] hash = new int[hashInts];
for (int j = 0; j < hashInts; j++) {
hash[j] = blockData[offset++];
}
- int indexInFile = blockData[offset++];
- int firstLineNumber = blockData[offset++];
- int lastLineNumber = blockData[offset++];
- int startUnit = blockData[offset++];
- int endUnit = blockData[offset];
-
- Block block = blockBuilder
- .setResourceId(resourceId)
- .setBlockHash(new ByteArray(hash))
- .setIndexInFile(indexInFile)
- .setLines(firstLineNumber, lastLineNumber)
- .setUnit(startUnit, endUnit)
- .build();
- result.add(block);
+ blockHash = new ByteArray(hash);
+ } else {
+ blockHash = byteHash;
+ offset += hashInts;
+ }
- index++;
- realIndex = resourceIdsIndex[index];
+ int indexInFile = blockData[offset++];
+ int firstLineNumber = blockData[offset++];
+ int lastLineNumber = blockData[offset++];
+ int startUnit = blockData[offset++];
+ int endUnit = blockData[offset];
+
+ return blockBuilder
+ .setResourceId(resourceId)
+ .setBlockHash(blockHash)
+ .setIndexInFile(indexInFile)
+ .setLines(firstLineNumber, lastLineNumber)
+ .setUnit(startUnit, endUnit)
+ .build();
+ }
+
+ private Block getBlock(int index, String resourceId) {
+ return createBlock(index, resourceId, null);
+ }
+
+ private class ResourceIterator implements Iterator<ResourceBlocks> {
+ private int index = 0;
+
+ @Override
+ public boolean hasNext() {
+ return index < size;
}
- return result;
+
+ @Override
+ public ResourceBlocks next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+
+ String resourceId = resourceIds[resourceIdsIndex[index]];
+ List<Block> blocks = new ArrayList<>();
+
+ // while we are at the same resource, keep going
+ do {
+ blocks.add(getBlock(resourceIdsIndex[index], resourceId));
+ index++;
+ } while (hasNext() && FastStringComparator.INSTANCE.compare(resourceIds[resourceIdsIndex[index]], resourceId) == 0);
+
+ return new ResourceBlocks(resourceId, blocks);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static class ResourceBlocks {
+ private Collection<Block> blocks;
+ private String resourceId;
+
+ public ResourceBlocks(String resourceId, Collection<Block> blocks) {
+ this.resourceId = resourceId;
+ this.blocks = blocks;
+ }
+
+ public Collection<Block> blocks() {
+ return blocks;
+ }
+
+ public String resourceId() {
+ return resourceId;
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Iterator<ResourceBlocks> iterator() {
+ ensureSorted();
+ return new ResourceIterator();
}
/**
while (index < size && !isLessByHash(size, index)) {
// extract block (note that there is no need to extract hash)
String resourceId = resourceIds[index];
- offset = index * blockInts + hashInts;
- int indexInFile = blockData[offset++];
- int firstLineNumber = blockData[offset++];
- int lastLineNumber = blockData[offset++];
- int startUnit = blockData[offset++];
- int endUnit = blockData[offset];
-
- Block block = blockBuilder
- .setResourceId(resourceId)
- .setBlockHash(sequenceHash)
- .setIndexInFile(indexInFile)
- .setLines(firstLineNumber, lastLineNumber)
- .setUnit(startUnit, endUnit)
- .build();
- result.add(block);
+ result.add(createBlock(index, resourceId, sequenceHash));
index++;
}
return result;
import org.junit.Test;
import org.sonar.duplications.block.Block;
import org.sonar.duplications.block.ByteArray;
+import org.sonar.duplications.index.PackedMemoryCloneIndex.ResourceBlocks;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Iterator;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.sameInstance;
import static org.junit.Assert.assertThat;
assertThat(block.getBlockHash(), sameInstance(requestedHash));
}
}
+
+ @Test
+ public void iterate() {
+ index.insert(newBlock("a", 1));
+ index.insert(newBlock("c", 1));
+ index.insert(newBlock("b", 1));
+ index.insert(newBlock("c", 2));
+ index.insert(newBlock("a", 2));
+
+ Iterator<ResourceBlocks> it = index.iterator();
+
+ ArrayList<ResourceBlocks> resourcesBlocks = new ArrayList<>();
+
+ while(it.hasNext()) {
+ resourcesBlocks.add(it.next());
+ }
+
+ assertThat(resourcesBlocks).hasSize(3);
+
+ assertThat(resourcesBlocks.get(0).resourceId()).isEqualTo("a");
+ assertThat(resourcesBlocks.get(1).resourceId()).isEqualTo("b");
+ assertThat(resourcesBlocks.get(2).resourceId()).isEqualTo("c");
+
+ assertThat(resourcesBlocks.get(0).blocks()).hasSize(2);
+ assertThat(resourcesBlocks.get(1).blocks()).hasSize(1);
+ assertThat(resourcesBlocks.get(2).blocks()).hasSize(2);
+
+ }
/**
* Given: index with initial capacity 1.
projects / sonarqube.git / commitdiff