From 29f0f0de0103bbae318cfb2cc324b2ff360c2cf4 Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Thu, 12 Nov 2015 09:45:11 +0100 Subject: [PATCH] SONAR-6993 Compute cross project duplications --- .../analysis/AnalysisMetadataHolderImpl.java | 21 +- ...ReportComputeEngineContainerPopulator.java | 6 + .../CrossProjectDuplicationStatusHolder.java | 41 +++ ...ossProjectDuplicationStatusHolderImpl.java | 69 ++++ .../IntegrateCrossProjectDuplications.java | 155 +++++++++ ...rossProjectDuplicationsRepositoryStep.java | 173 +++++++++ .../step/ReportComputationSteps.java | 2 + .../computation/util/InitializedProperty.java | 44 +++ .../analysis/AnalysisMetadataHolderRule.java | 106 ++++++ ...rojectDuplicationStatusHolderImplTest.java | 111 ++++++ ...IntegrateCrossProjectDuplicationsTest.java | 329 ++++++++++++++++++ ...ProjectDuplicationsRepositoryStepTest.java | 252 ++++++++++++++ 12 files changed, 1289 insertions(+), 20 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolder.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolderImpl.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/duplication/IntegrateCrossProjectDuplications.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/step/LoadCrossProjectDuplicationsRepositoryStep.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/util/InitializedProperty.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/analysis/AnalysisMetadataHolderRule.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolderImplTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/duplication/IntegrateCrossProjectDuplicationsTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/step/LoadCrossProjectDuplicationsRepositoryStepTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/analysis/AnalysisMetadataHolderImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/analysis/AnalysisMetadataHolderImpl.java index 0c1001e4e99..db8e5da074c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/analysis/AnalysisMetadataHolderImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/analysis/AnalysisMetadataHolderImpl.java @@ -23,6 +23,7 @@ import java.util.Date; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.sonar.server.computation.snapshot.Snapshot; +import org.sonar.server.computation.util.InitializedProperty; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; @@ -111,24 +112,4 @@ public class AnalysisMetadataHolderImpl implements MutableAnalysisMetadataHolder return rootComponentRef.getProperty(); } - private static class InitializedProperty { - private E property; - private boolean initialized = false; - - public InitializedProperty setProperty(@Nullable E property) { - this.property = property; - this.initialized = true; - return this; - } - - @CheckForNull - public E getProperty() { - return property; - } - - public boolean isInitialized() { - return initialized; - } - } - } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/container/ReportComputeEngineContainerPopulator.java b/server/sonar-server/src/main/java/org/sonar/server/computation/container/ReportComputeEngineContainerPopulator.java index b625ee51faf..26ced5c2c37 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/container/ReportComputeEngineContainerPopulator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/container/ReportComputeEngineContainerPopulator.java @@ -30,7 +30,9 @@ import org.sonar.server.computation.component.DbIdsRepositoryImpl; import org.sonar.server.computation.component.SettingsRepositoryImpl; import org.sonar.server.computation.component.TreeRootHolderImpl; import org.sonar.server.computation.debt.DebtModelHolderImpl; +import org.sonar.server.computation.duplication.CrossProjectDuplicationStatusHolderImpl; import org.sonar.server.computation.duplication.DuplicationRepositoryImpl; +import org.sonar.server.computation.duplication.IntegrateCrossProjectDuplications; import org.sonar.server.computation.event.EventRepositoryImpl; import org.sonar.server.computation.filesystem.ComputationTempFolderProvider; import org.sonar.server.computation.issue.BaseIssuesLoader; @@ -116,6 +118,7 @@ public final class ReportComputeEngineContainerPopulator implements ContainerPop // holders AnalysisMetadataHolderImpl.class, + CrossProjectDuplicationStatusHolderImpl.class, BatchReportDirectoryHolderImpl.class, TreeRootHolderImpl.class, PeriodsHolderImpl.class, @@ -185,6 +188,9 @@ public final class ReportComputeEngineContainerPopulator implements ContainerPop TrackerExecution.class, BaseIssuesLoader.class, + // duplication + IntegrateCrossProjectDuplications.class, + // views ViewIndex.class); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolder.java b/server/sonar-server/src/main/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolder.java new file mode 100644 index 00000000000..89229d9b36e --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolder.java @@ -0,0 +1,41 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.duplication; + +import org.sonar.server.computation.analysis.AnalysisMetadataHolder; + +/** + * A simple holder to know if the cross project duplication should be computed or not. + * + * A log will be displayed whether or not the cross project duplication is enabled or not. + */ +public interface CrossProjectDuplicationStatusHolder { + + /** + * Cross project duplications is enabled when : + * + *
  • {@link AnalysisMetadataHolder#isCrossProjectDuplicationEnabled()} is true
    • + *
  • {@link AnalysisMetadataHolder#getBranch()} ()} is null
    • + *     + */ + boolean isEnabled(); + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolderImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolderImpl.java new file mode 100644 index 00000000000..f9896d279a0 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolderImpl.java @@ -0,0 +1,69 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.duplication; + +import com.google.common.base.Preconditions; +import javax.annotation.CheckForNull; +import org.picocontainer.Startable; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.server.computation.analysis.AnalysisMetadataHolder; + +public class CrossProjectDuplicationStatusHolderImpl implements CrossProjectDuplicationStatusHolder, Startable { + + private static final Logger LOGGER = Loggers.get(CrossProjectDuplicationStatusHolderImpl.class); + + @CheckForNull + private Boolean enabled; + private final AnalysisMetadataHolder analysisMetadataHolder; + + public CrossProjectDuplicationStatusHolderImpl(AnalysisMetadataHolder analysisMetadataHolder) { + this.analysisMetadataHolder = analysisMetadataHolder; + } + + @Override + public boolean isEnabled() { + Preconditions.checkState(enabled != null, "Flag hasn't been initialized, the start() should have been called before."); + return enabled; + } + + @Override + public void start() { + boolean crossProjectDuplicationIsEnabledInReport = analysisMetadataHolder.isCrossProjectDuplicationEnabled(); + boolean branchIsUsed = analysisMetadataHolder.getBranch() != null; + if (crossProjectDuplicationIsEnabledInReport && !branchIsUsed) { + LOGGER.info("Cross project duplication is enabled"); + this.enabled = true; + } else { + if (!crossProjectDuplicationIsEnabledInReport) { + LOGGER.info("Cross project duplication is disabled because it's disabled in the analysis report"); + } else { + LOGGER.info("Cross project duplication is disabled because of a branch is used"); + } + this.enabled = false; + } + } + + @Override + public void stop() { + // nothing to do + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/duplication/IntegrateCrossProjectDuplications.java b/server/sonar-server/src/main/java/org/sonar/server/computation/duplication/IntegrateCrossProjectDuplications.java new file mode 100644 index 00000000000..6d1ce20b839 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/duplication/IntegrateCrossProjectDuplications.java @@ -0,0 +1,155 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.duplication; + +import com.google.common.base.Predicate; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import org.sonar.api.config.Settings; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.duplications.block.Block; +import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm; +import org.sonar.duplications.index.CloneGroup; +import org.sonar.duplications.index.CloneIndex; +import org.sonar.duplications.index.ClonePart; +import org.sonar.duplications.index.PackedMemoryCloneIndex; +import org.sonar.server.computation.component.Component; + +import static com.google.common.collect.FluentIterable.from; + +/** + * Transform a list of duplication blocks into clone groups, then add these clone groups into the duplication repository. + */ +public class IntegrateCrossProjectDuplications { + + private static final Logger LOGGER = Loggers.get(IntegrateCrossProjectDuplications.class); + + private static final String JAVA_KEY = "java"; + + private static final int MAX_CLONE_GROUP_PER_FILE = 100; + private static final int MAX_CLONE_PART_PER_GROUP = 100; + + private final Settings settings; + private final DuplicationRepository duplicationRepository; + + private Map numberOfUnitsByLanguage = new HashMap<>(); + + public IntegrateCrossProjectDuplications(Settings settings, DuplicationRepository duplicationRepository) { + this.settings = settings; + this.duplicationRepository = duplicationRepository; + } + + public void computeCpd(Component component, Collection originBlocks, Collection duplicationBlocks) { + CloneIndex duplicationIndex = new PackedMemoryCloneIndex(); + populateIndex(duplicationIndex, originBlocks); + populateIndex(duplicationIndex, duplicationBlocks); + + List duplications = SuffixTreeCloneDetectionAlgorithm.detect(duplicationIndex, originBlocks); + Iterable filtered = from(duplications).filter(getNumberOfUnitsNotLessThan(component.getFileAttributes().getLanguageKey())); + addDuplications(component, filtered); + } + + private static void populateIndex(CloneIndex duplicationIndex, Collection duplicationBlocks) { + for (Block block : duplicationBlocks) { + duplicationIndex.insert(block); + } + } + + private void addDuplications(Component file, Iterable duplications) { + int cloneGroupCount = 0; + for (CloneGroup duplication : duplications) { + cloneGroupCount++; + if (cloneGroupCount > MAX_CLONE_GROUP_PER_FILE) { + LOGGER.warn("Too many duplication groups on file {}. Keeping only the first {} groups.", file.getKey(), MAX_CLONE_GROUP_PER_FILE); + break; + } + addDuplication(file, duplication); + } + } + + private void addDuplication(Component file, CloneGroup duplication) { + ClonePart originPart = duplication.getOriginPart(); + TextBlock originTextBlock = new TextBlock(originPart.getStartLine(), originPart.getEndLine()); + int clonePartCount = 0; + for (ClonePart part : from(duplication.getCloneParts()).filter(new DoesNotMatchOriginalPart(originPart))) { + clonePartCount++; + if (clonePartCount > MAX_CLONE_PART_PER_GROUP) { + LOGGER.warn("Too many duplication references on file {} for block at line {}. Keeping only the first {} references.", + file.getKey(), originPart.getStartLine(), MAX_CLONE_PART_PER_GROUP); + break; + } + duplicationRepository.addDuplication(file, originTextBlock, part.getResourceId(), + new TextBlock(part.getStartLine(), part.getEndLine())); + } + } + + private NumberOfUnitsNotLessThan getNumberOfUnitsNotLessThan(String language) { + NumberOfUnitsNotLessThan numberOfUnitsNotLessThan = numberOfUnitsByLanguage.get(language); + if (numberOfUnitsNotLessThan == null) { + numberOfUnitsNotLessThan = new NumberOfUnitsNotLessThan(getMinimumTokens(language)); + numberOfUnitsByLanguage.put(language, numberOfUnitsNotLessThan); + } + return numberOfUnitsNotLessThan; + } + + private int getMinimumTokens(String languageKey) { + // The java language is an exception : it doesn't compute tokens but statement, so the settings could not be used. + if (languageKey.equalsIgnoreCase(JAVA_KEY)) { + return 0; + } + int minimumTokens = settings.getInt("sonar.cpd." + languageKey + ".minimumTokens"); + if (minimumTokens == 0) { + return 100; + } + return minimumTokens; + } + + private static class NumberOfUnitsNotLessThan implements Predicate { + private final int min; + + public NumberOfUnitsNotLessThan(int min) { + this.min = min; + } + + @Override + public boolean apply(@Nonnull CloneGroup input) { + return input.getLengthInUnits() >= min; + } + } + + private static class DoesNotMatchOriginalPart implements Predicate { + private final ClonePart originPart; + + private DoesNotMatchOriginalPart(ClonePart originPart) { + this.originPart = originPart; + } + + @Override + public boolean apply(ClonePart part) { + return !part.equals(originPart); + } + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/LoadCrossProjectDuplicationsRepositoryStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/LoadCrossProjectDuplicationsRepositoryStep.java new file mode 100644 index 00000000000..b351cc6b345 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/LoadCrossProjectDuplicationsRepositoryStep.java @@ -0,0 +1,173 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.step; + +import com.google.common.base.Function; +import java.util.Collection; +import java.util.List; +import javax.annotation.Nonnull; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.batch.protocol.output.BatchReport.CpdTextBlock; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.duplication.DuplicationUnitDto; +import org.sonar.duplications.block.Block; +import org.sonar.duplications.block.ByteArray; +import org.sonar.server.computation.analysis.AnalysisMetadataHolder; +import org.sonar.server.computation.batch.BatchReportReader; +import org.sonar.server.computation.component.Component; +import org.sonar.server.computation.component.CrawlerDepthLimit; +import org.sonar.server.computation.component.DepthTraversalTypeAwareCrawler; +import org.sonar.server.computation.component.TreeRootHolder; +import org.sonar.server.computation.component.TypeAwareVisitorAdapter; +import org.sonar.server.computation.duplication.CrossProjectDuplicationStatusHolder; +import org.sonar.server.computation.duplication.IntegrateCrossProjectDuplications; +import org.sonar.server.computation.snapshot.Snapshot; + +import static com.google.common.collect.FluentIterable.from; +import static com.google.common.collect.Lists.newArrayList; +import static org.sonar.server.computation.component.ComponentVisitor.Order.PRE_ORDER; + +/** + * Feed the duplications repository from the cross project duplication blocks computed with duplications blocks of the analysis report. + * + * Blocks can be empty if : + * - The file is excluded from the analysis using {@link org.sonar.api.CoreProperties#CPD_EXCLUSIONS} + * - On Java, if the number of statements of the file is too small, nothing will be sent. + */ +public class LoadCrossProjectDuplicationsRepositoryStep implements ComputationStep { + + private static final Logger LOGGER = Loggers.get(LoadCrossProjectDuplicationsRepositoryStep.class); + + private final TreeRootHolder treeRootHolder; + private final BatchReportReader reportReader; + private final AnalysisMetadataHolder analysisMetadataHolder; + private final IntegrateCrossProjectDuplications integrateCrossProjectDuplications; + private final CrossProjectDuplicationStatusHolder crossProjectDuplicationStatusHolder; + private final DbClient dbClient; + + public LoadCrossProjectDuplicationsRepositoryStep(TreeRootHolder treeRootHolder, BatchReportReader reportReader, + AnalysisMetadataHolder analysisMetadataHolder, CrossProjectDuplicationStatusHolder crossProjectDuplicationStatusHolder, + IntegrateCrossProjectDuplications integrateCrossProjectDuplications, DbClient dbClient) { + this.treeRootHolder = treeRootHolder; + this.reportReader = reportReader; + this.analysisMetadataHolder = analysisMetadataHolder; + this.integrateCrossProjectDuplications = integrateCrossProjectDuplications; + this.crossProjectDuplicationStatusHolder = crossProjectDuplicationStatusHolder; + this.dbClient = dbClient; + } + + @Override + public void execute() { + if (crossProjectDuplicationStatusHolder.isEnabled()) { + new DepthTraversalTypeAwareCrawler(new CrossProjectDuplicationVisitor()).visit(treeRootHolder.getRoot()); + } + } + + @Override + public String getDescription() { + return "Compute cross project duplications"; + } + + private class CrossProjectDuplicationVisitor extends TypeAwareVisitorAdapter { + + private CrossProjectDuplicationVisitor() { + super(CrawlerDepthLimit.FILE, PRE_ORDER); + } + + @Override + public void visitFile(Component file) { + visitComponent(file); + } + + private void visitComponent(Component component) { + List cpdTextBlocks = newArrayList(reportReader.readCpdTextBlocks(component.getReportAttributes().getRef())); + LOGGER.trace("Found {} cpd blocks on file {}", cpdTextBlocks.size(), component.getKey()); + if (cpdTextBlocks.isEmpty()) { + return; + } + + Collection hashes = from(cpdTextBlocks).transform(CpdTextBlockToHash.INSTANCE).toList(); + List dtos = selectDuplicates(component, hashes); + Collection duplicatedBlocks = from(dtos).transform(DtoToBlock.INSTANCE).toList(); + Collection originBlocks = from(cpdTextBlocks).transform(new CpdTextBlockToBlock(component.getKey())).toList(); + + integrateCrossProjectDuplications.computeCpd(component, originBlocks, duplicatedBlocks); + } + + private List selectDuplicates(Component file, Collection hashes) { + DbSession dbSession = dbClient.openSession(false); + try { + Snapshot projectSnapshot = analysisMetadataHolder.getBaseProjectSnapshot(); + Long projectSnapshotId = projectSnapshot == null ? null : projectSnapshot.getId(); + return dbClient.duplicationDao().selectCandidates(dbSession, projectSnapshotId, file.getFileAttributes().getLanguageKey(), hashes); + } finally { + dbClient.closeSession(dbSession); + } + } + } + + private enum CpdTextBlockToHash implements Function { + INSTANCE; + + @Override + public String apply(@Nonnull CpdTextBlock duplicationBlock) { + return duplicationBlock.getHash(); + } + } + + private enum DtoToBlock implements Function { + INSTANCE; + + @Override + public Block apply(@Nonnull DuplicationUnitDto dto) { + // Not that the dto doesn't contains start/end token indexes + return Block.builder() + .setResourceId(dto.getComponentKey()) + .setBlockHash(new ByteArray(dto.getHash())) + .setIndexInFile(dto.getIndexInFile()) + .setLines(dto.getStartLine(), dto.getEndLine()) + .build(); + } + } + + private static class CpdTextBlockToBlock implements Function { + private final String fileKey; + private int indexInFile = 0; + + public CpdTextBlockToBlock(String fileKey) { + this.fileKey = fileKey; + } + + @Override + public Block apply(@Nonnull CpdTextBlock duplicationBlock) { + return Block.builder() + .setResourceId(fileKey) + .setBlockHash(new ByteArray(duplicationBlock.getHash())) + .setIndexInFile(indexInFile++) + .setLines(duplicationBlock.getStartLine(), duplicationBlock.getEndLine()) + .setUnit(duplicationBlock.getStartTokenIndex(), duplicationBlock.getEndTokenIndex()) + .build(); + } + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/ReportComputationSteps.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/ReportComputationSteps.java index e298bf36e4c..74413a1057d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/step/ReportComputationSteps.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/ReportComputationSteps.java @@ -53,6 +53,8 @@ public class ReportComputationSteps implements ComputationSteps { LoadQualityGateStep.class, LoadPeriodsStep.class, + LoadCrossProjectDuplicationsRepositoryStep.class, + // data computation SizeMeasuresStep.class, NewCoverageMeasuresStep.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/util/InitializedProperty.java b/server/sonar-server/src/main/java/org/sonar/server/computation/util/InitializedProperty.java new file mode 100644 index 00000000000..88f2f906c51 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/util/InitializedProperty.java @@ -0,0 +1,44 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.util; + +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; + +public class InitializedProperty { + private E property; + private boolean initialized = false; + + public InitializedProperty setProperty(@Nullable E property) { + this.property = property; + this.initialized = true; + return this; + } + + @CheckForNull + public E getProperty() { + return property; + } + + public boolean isInitialized() { + return initialized; + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/analysis/AnalysisMetadataHolderRule.java b/server/sonar-server/src/test/java/org/sonar/server/computation/analysis/AnalysisMetadataHolderRule.java new file mode 100644 index 00000000000..f4adc2669a5 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/analysis/AnalysisMetadataHolderRule.java @@ -0,0 +1,106 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.analysis; + +import java.util.Date; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import org.junit.rules.ExternalResource; +import org.sonar.server.computation.snapshot.Snapshot; +import org.sonar.server.computation.util.InitializedProperty; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + +public class AnalysisMetadataHolderRule extends ExternalResource implements AnalysisMetadataHolder { + + private InitializedProperty analysisDate = new InitializedProperty<>(); + + private InitializedProperty baseProjectSnapshot = new InitializedProperty<>(); + + private InitializedProperty crossProjectDuplicationEnabled = new InitializedProperty<>(); + + private InitializedProperty branch = new InitializedProperty<>(); + + private InitializedProperty rootComponentRef = new InitializedProperty<>(); + + public AnalysisMetadataHolderRule setAnalysisDate(Date date) { + checkNotNull(date, "Date must not be null"); + this.analysisDate.setProperty(date.getTime()); + return this; + } + + @Override + public Date getAnalysisDate() { + checkState(analysisDate.isInitialized(), "Analysis date has not been set"); + return new Date(this.analysisDate.getProperty()); + } + + @Override + public boolean isFirstAnalysis() { + return getBaseProjectSnapshot() == null; + } + + public AnalysisMetadataHolderRule setBaseProjectSnapshot(@Nullable Snapshot baseProjectSnapshot) { + this.baseProjectSnapshot.setProperty(baseProjectSnapshot); + return this; + } + + @Override + @CheckForNull + public Snapshot getBaseProjectSnapshot() { + checkState(baseProjectSnapshot.isInitialized(), "Base project snapshot has not been set"); + return baseProjectSnapshot.getProperty(); + } + + public AnalysisMetadataHolderRule setCrossProjectDuplicationEnabled(boolean isCrossProjectDuplicationEnabled) { + this.crossProjectDuplicationEnabled.setProperty(isCrossProjectDuplicationEnabled); + return this; + } + + @Override + public boolean isCrossProjectDuplicationEnabled() { + checkState(crossProjectDuplicationEnabled.isInitialized(), "Cross project duplication flag has not been set"); + return crossProjectDuplicationEnabled.getProperty(); + } + + public AnalysisMetadataHolderRule setBranch(@Nullable String branch) { + this.branch.setProperty(branch); + return this; + } + + @Override + public String getBranch() { + checkState(branch.isInitialized(), "Branch has not been set"); + return branch.getProperty(); + } + + public AnalysisMetadataHolderRule setRootComponentRef(int rootComponentRef) { + this.rootComponentRef.setProperty(rootComponentRef); + return this; + } + + @Override + public int getRootComponentRef() { + checkState(rootComponentRef.isInitialized(), "Root component ref has not been set"); + return rootComponentRef.getProperty(); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolderImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolderImplTest.java new file mode 100644 index 00000000000..d17bea2edcc --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/duplication/CrossProjectDuplicationStatusHolderImplTest.java @@ -0,0 +1,111 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.duplication; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.server.computation.analysis.AnalysisMetadataHolderRule; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CrossProjectDuplicationStatusHolderImplTest { + + static String BRANCH = "origin/master"; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Rule + public LogTester logTester = new LogTester(); + + @Rule + public AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule(); + + CrossProjectDuplicationStatusHolderImpl underTest = new CrossProjectDuplicationStatusHolderImpl(analysisMetadataHolder); + + @Test + public void cross_project_duplication_is_enabled_when_enabled_in_report_and_no_branch() throws Exception { + analysisMetadataHolder + .setCrossProjectDuplicationEnabled(true) + .setBranch(null); + underTest.start(); + + assertThat(underTest.isEnabled()).isTrue(); + assertThat(logTester.logs(LoggerLevel.INFO)).containsOnly("Cross project duplication is enabled"); + } + + @Test + public void cross_project_duplication_is_disabled_when_not_enabled_in_report() throws Exception { + analysisMetadataHolder + .setCrossProjectDuplicationEnabled(false) + .setBranch(null); + underTest.start(); + + assertThat(underTest.isEnabled()).isFalse(); + assertThat(logTester.logs(LoggerLevel.INFO)).containsOnly("Cross project duplication is disabled because it's disabled in the analysis report"); + } + + @Test + public void cross_project_duplication_is_disabled_when_branch_is_used() throws Exception { + analysisMetadataHolder + .setCrossProjectDuplicationEnabled(true) + .setBranch(BRANCH); + underTest.start(); + + assertThat(underTest.isEnabled()).isFalse(); + assertThat(logTester.logs(LoggerLevel.INFO)).containsOnly("Cross project duplication is disabled because of a branch is used"); + } + + @Test + public void cross_project_duplication_is_disabled_when_not_enabled_in_report_and_when_branch_is_used() throws Exception { + analysisMetadataHolder + .setCrossProjectDuplicationEnabled(false) + .setBranch(BRANCH); + underTest.start(); + + assertThat(underTest.isEnabled()).isFalse(); + assertThat(logTester.logs(LoggerLevel.INFO)).containsOnly("Cross project duplication is disabled because it's disabled in the analysis report"); + } + + @Test + public void flag_is_build_in_start() throws Exception { + analysisMetadataHolder + .setCrossProjectDuplicationEnabled(true) + .setBranch(null); + underTest.start(); + assertThat(underTest.isEnabled()).isTrue(); + + // Change the boolean from the report. This can never happen, it's only to validate that the flag is build in the start method + analysisMetadataHolder.setCrossProjectDuplicationEnabled(false); + assertThat(underTest.isEnabled()).isTrue(); + } + + @Test + public void isEnabled_throws_ISE_when_start_have_not_been_called_before() throws Exception { + thrown.expect(IllegalStateException.class); + thrown.expectMessage("Flag hasn't been initialized, the start() should have been called before"); + + underTest.isEnabled(); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/duplication/IntegrateCrossProjectDuplicationsTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/duplication/IntegrateCrossProjectDuplicationsTest.java new file mode 100644 index 00000000000..638d77b8859 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/duplication/IntegrateCrossProjectDuplicationsTest.java @@ -0,0 +1,329 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.duplication; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.config.Settings; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.duplications.block.Block; +import org.sonar.duplications.block.ByteArray; +import org.sonar.server.computation.component.Component; +import org.sonar.server.computation.component.FileAttributes; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.sonar.server.computation.component.Component.Type.FILE; +import static org.sonar.server.computation.component.ReportComponent.builder; + +public class IntegrateCrossProjectDuplicationsTest { + + @Rule + public LogTester logTester = new LogTester(); + + static final String XOO_LANGUAGE = "xoo"; + + static final String ORIGIN_FILE_KEY = "ORIGIN_FILE_KEY"; + static final Component ORIGIN_FILE = builder(FILE, 1) + .setKey(ORIGIN_FILE_KEY) + .setFileAttributes(new FileAttributes(false, XOO_LANGUAGE)) + .build(); + + static final String OTHER_FILE_KEY = "OTHER_FILE_KEY"; + + Settings settings = new Settings(); + + DuplicationRepository duplicationRepository = mock(DuplicationRepository.class); + + IntegrateCrossProjectDuplications underTest = new IntegrateCrossProjectDuplications(settings, duplicationRepository); + + @Test + public void add_duplications_from_two_blocks() { + settings.setProperty("sonar.cpd.xoo.minimumTokens", 10); + + Collection originBlocks = asList( + new Block.Builder() + .setResourceId(ORIGIN_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(30, 43) + .setUnit(0, 5) + .build(), + new Block.Builder() + .setResourceId(ORIGIN_FILE_KEY) + .setBlockHash(new ByteArray("2b5747f0e4c59124")) + .setIndexInFile(1) + .setLines(32, 45) + .setUnit(5, 20) + .build() + ); + + Collection duplicatedBlocks = asList( + new Block.Builder() + .setResourceId(OTHER_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(40, 53) + .build(), + new Block.Builder() + .setResourceId(OTHER_FILE_KEY) + .setBlockHash(new ByteArray("2b5747f0e4c59124")) + .setIndexInFile(1) + .setLines(42, 55) + .build()); + + underTest.computeCpd(ORIGIN_FILE, originBlocks, duplicatedBlocks); + + verify(duplicationRepository).addDuplication( + ORIGIN_FILE, + new TextBlock(30, 45), + OTHER_FILE_KEY, + new TextBlock(40, 55) + ); + } + + @Test + public void add_duplications_from_a_single_block() { + settings.setProperty("sonar.cpd.xoo.minimumTokens", 10); + + Collection originBlocks = singletonList( + // This block contains 11 tokens -> a duplication will be created + new Block.Builder() + .setResourceId(ORIGIN_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(30, 45) + .setUnit(0, 10) + .build() + ); + + Collection duplicatedBlocks = singletonList( + new Block.Builder() + .setResourceId(OTHER_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(40, 55) + .build() + ); + + underTest.computeCpd(ORIGIN_FILE, originBlocks, duplicatedBlocks); + + verify(duplicationRepository).addDuplication( + ORIGIN_FILE, + new TextBlock(30, 45), + OTHER_FILE_KEY, + new TextBlock(40, 55) + ); + } + + @Test + public void add_no_duplication_when_not_enough_tokens() { + settings.setProperty("sonar.cpd.xoo.minimumTokens", 10); + + Collection originBlocks = singletonList( + // This block contains 5 tokens -> not enough to consider it as a duplication + new Block.Builder() + .setResourceId(ORIGIN_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(30, 45) + .setUnit(0, 4) + .build() + ); + + Collection duplicatedBlocks = singletonList( + new Block.Builder() + .setResourceId(OTHER_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(40, 55) + .build() + ); + + underTest.computeCpd(ORIGIN_FILE, originBlocks, duplicatedBlocks); + + verifyNoMoreInteractions(duplicationRepository); + } + + @Test + public void add_no_duplication_when_no_duplicated_blocks() { + settings.setProperty("sonar.cpd.xoo.minimumTokens", 10); + + Collection originBlocks = singletonList( + new Block.Builder() + .setResourceId(ORIGIN_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(30, 45) + .setUnit(0, 10) + .build() + ); + + underTest.computeCpd(ORIGIN_FILE, originBlocks, Collections.emptyList()); + + verifyNoMoreInteractions(duplicationRepository); + } + + @Test + public void add_duplication_for_java_even_when_no_token() { + Component javaFile = builder(FILE, 1) + .setKey(ORIGIN_FILE_KEY) + .setFileAttributes(new FileAttributes(false, "java")) + .build(); + + Collection originBlocks = singletonList( + // This block contains 0 token + new Block.Builder() + .setResourceId(ORIGIN_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(30, 45) + .setUnit(0, 0) + .build() + ); + + Collection duplicatedBlocks = singletonList( + new Block.Builder() + .setResourceId(OTHER_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(40, 55) + .build() + ); + + underTest.computeCpd(javaFile, originBlocks, duplicatedBlocks); + + verify(duplicationRepository).addDuplication( + ORIGIN_FILE, + new TextBlock(30, 45), + OTHER_FILE_KEY, + new TextBlock(40, 55) + ); + } + + @Test + public void default_minimum_tokens_is_one_hundred() { + settings.setProperty("sonar.cpd.xoo.minimumTokens", (Integer) null); + + Collection originBlocks = singletonList( + new Block.Builder() + .setResourceId(ORIGIN_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(30, 45) + .setUnit(0, 100) + .build() + ); + + Collection duplicatedBlocks = singletonList( + new Block.Builder() + .setResourceId(OTHER_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(40, 55) + .build() + ); + + underTest.computeCpd(ORIGIN_FILE, originBlocks, duplicatedBlocks); + + verify(duplicationRepository).addDuplication( + ORIGIN_FILE, + new TextBlock(30, 45), + OTHER_FILE_KEY, + new TextBlock(40, 55) + ); + } + + @Test + public void do_not_compute_more_than_one_hundred_duplications_when_too_many_duplicated_references() throws Exception { + Collection originBlocks = new ArrayList<>(); + Collection duplicatedBlocks = new ArrayList<>(); + + Block.Builder blockBuilder = new Block.Builder() + .setResourceId(ORIGIN_FILE_KEY) + .setBlockHash(new ByteArray("a8998353e96320ec")) + .setIndexInFile(0) + .setLines(30, 45) + .setUnit(0, 100); + originBlocks.add(blockBuilder.build()); + + // Generate more than 100 duplications of the same block + for (int i = 0; i < 110; i++) { + duplicatedBlocks.add( + blockBuilder + .setResourceId(randomAlphanumeric(16)) + .build() + ); + } + + underTest.computeCpd(ORIGIN_FILE, originBlocks, duplicatedBlocks); + + assertThat(logTester.logs(LoggerLevel.WARN)).containsOnly( + "Too many duplication references on file " + ORIGIN_FILE_KEY + " for block at line 30. Keeping only the first 100 references."); + verify(duplicationRepository, times(100)).addDuplication(eq(ORIGIN_FILE), any(TextBlock.class), anyString(), any(TextBlock.class)); + } + + @Test + public void do_not_compute_more_than_one_hundred_duplications_when_too_many_duplications() throws Exception { + Collection originBlocks = new ArrayList<>(); + Collection duplicatedBlocks = new ArrayList<>(); + + Block.Builder blockBuilder = new Block.Builder() + .setIndexInFile(0) + .setLines(30, 45) + .setUnit(0, 100); + + // Generate more than 100 duplication on different files + for (int i = 0; i < 110; i++) { + String hash = randomAlphanumeric(16); + originBlocks.add( + blockBuilder + .setResourceId(ORIGIN_FILE_KEY) + .setBlockHash(new ByteArray(hash)) + .build()); + duplicatedBlocks.add( + blockBuilder + .setResourceId(randomAlphanumeric(16)) + .setBlockHash(new ByteArray(hash)) + .build() + ); + } + + underTest.computeCpd(ORIGIN_FILE, originBlocks, duplicatedBlocks); + + verify(duplicationRepository, times(100)).addDuplication(eq(ORIGIN_FILE), any(TextBlock.class), anyString(), any(TextBlock.class)); + assertThat(logTester.logs(LoggerLevel.WARN)).containsOnly("Too many duplication groups on file " + ORIGIN_FILE_KEY + ". Keeping only the first 100 groups."); + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/step/LoadCrossProjectDuplicationsRepositoryStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/step/LoadCrossProjectDuplicationsRepositoryStepTest.java new file mode 100644 index 00000000000..e643e5adfb3 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/step/LoadCrossProjectDuplicationsRepositoryStepTest.java @@ -0,0 +1,252 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.step; + +import java.util.Arrays; +import java.util.Collections; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.utils.System2; +import org.sonar.batch.protocol.output.BatchReport; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ComponentTesting; +import org.sonar.db.component.SnapshotDto; +import org.sonar.db.component.SnapshotTesting; +import org.sonar.db.duplication.DuplicationUnitDto; +import org.sonar.duplications.block.Block; +import org.sonar.duplications.block.ByteArray; +import org.sonar.server.computation.analysis.AnalysisMetadataHolderRule; +import org.sonar.server.computation.batch.BatchReportReaderRule; +import org.sonar.server.computation.batch.TreeRootHolderRule; +import org.sonar.server.computation.component.Component; +import org.sonar.server.computation.component.FileAttributes; +import org.sonar.server.computation.component.ReportComponent; +import org.sonar.server.computation.duplication.CrossProjectDuplicationStatusHolder; +import org.sonar.server.computation.duplication.IntegrateCrossProjectDuplications; +import org.sonar.server.computation.snapshot.Snapshot; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; +import static org.sonar.server.computation.component.Component.Type.FILE; +import static org.sonar.server.computation.component.Component.Type.PROJECT; + +public class LoadCrossProjectDuplicationsRepositoryStepTest { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + static final String XOO_LANGUAGE = "xoo"; + + static final int PROJECT_REF = 1; + static final int FILE_REF = 2; + static final String CURRENT_FILE_KEY = "FILE_KEY"; + + static final Component CURRENT_FILE = ReportComponent.builder(FILE, FILE_REF) + .setKey(CURRENT_FILE_KEY) + .setFileAttributes(new FileAttributes(false, XOO_LANGUAGE)) + .build(); + + @Rule + public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule().setRoot( + ReportComponent.builder(PROJECT, PROJECT_REF) + .addChildren(CURRENT_FILE + ).build()); + + @Rule + public BatchReportReaderRule batchReportReader = new BatchReportReaderRule(); + + @Rule + public AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule(); + + CrossProjectDuplicationStatusHolder crossProjectDuplicationStatusHolder = mock(CrossProjectDuplicationStatusHolder.class); + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + DbClient dbClient = dbTester.getDbClient(); + + DbSession dbSession = dbTester.getSession(); + + IntegrateCrossProjectDuplications integrateCrossProjectDuplications = mock(IntegrateCrossProjectDuplications.class); + + Snapshot baseProjectSnapshot; + + ComputationStep underTest = new LoadCrossProjectDuplicationsRepositoryStep(treeRootHolder, batchReportReader, analysisMetadataHolder, crossProjectDuplicationStatusHolder, + integrateCrossProjectDuplications, dbClient); + + @Before + public void setUp() throws Exception { + ComponentDto project = ComponentTesting.newProjectDto(); + dbClient.componentDao().insert(dbSession, project); + SnapshotDto projectSnapshot = SnapshotTesting.newSnapshotForProject(project); + dbClient.snapshotDao().insert(dbSession, projectSnapshot); + dbSession.commit(); + + baseProjectSnapshot = new Snapshot.Builder() + .setId(projectSnapshot.getId()) + .setCreatedAt(projectSnapshot.getCreatedAt()) + .build(); + } + + @Test + public void call_compute_cpd() throws Exception { + when(crossProjectDuplicationStatusHolder.isEnabled()).thenReturn(true); + analysisMetadataHolder.setBaseProjectSnapshot(baseProjectSnapshot); + + ComponentDto otherProject = createProject("OTHER_PROJECT_KEY"); + SnapshotDto otherProjectSnapshot = createProjectSnapshot(otherProject); + + ComponentDto otherFIle = createFile("OTHER_FILE_KEY", otherProject); + SnapshotDto otherFileSnapshot = createFileSnapshot(otherFIle, otherProjectSnapshot); + + String hash = "a8998353e96320ec"; + DuplicationUnitDto duplicate = new DuplicationUnitDto() + .setHash(hash) + .setStartLine(40) + .setEndLine(55) + .setIndexInFile(0) + .setProjectSnapshotId(otherProjectSnapshot.getId()) + .setSnapshotId(otherFileSnapshot.getId()); + dbClient.duplicationDao().insert(dbSession, singletonList(duplicate)); + dbSession.commit(); + + BatchReport.CpdTextBlock originBlock = BatchReport.CpdTextBlock.newBuilder() + .setHash(hash) + .setStartLine(30) + .setEndLine(45) + .setStartTokenIndex(0) + .setEndTokenIndex(10) + .build(); + batchReportReader.putDuplicationBlocks(FILE_REF, asList(originBlock)); + + underTest.execute(); + + verify(integrateCrossProjectDuplications).computeCpd(CURRENT_FILE, + Arrays.asList( + new Block.Builder() + .setResourceId(CURRENT_FILE_KEY) + .setBlockHash(new ByteArray(hash)) + .setIndexInFile(0) + .setLines(originBlock.getStartLine(), originBlock.getEndLine()) + .setUnit(originBlock.getStartTokenIndex(), originBlock.getEndTokenIndex()) + .build() + ), + Arrays.asList( + new Block.Builder() + .setResourceId(otherFIle.getKey()) + .setBlockHash(new ByteArray(hash)) + .setIndexInFile(duplicate.getIndexInFile()) + .setLines(duplicate.getStartLine(), duplicate.getEndLine()) + .build() + ) + ); + } + + @Test + public void nothing_to_do_when_cross_project_duplication_is_disabled() throws Exception { + when(crossProjectDuplicationStatusHolder.isEnabled()).thenReturn(false); + analysisMetadataHolder.setBaseProjectSnapshot(baseProjectSnapshot); + + ComponentDto otherProject = createProject("OTHER_PROJECT_KEY"); + SnapshotDto otherProjectSnapshot = createProjectSnapshot(otherProject); + + ComponentDto otherFIle = createFile("OTHER_FILE_KEY", otherProject); + SnapshotDto otherFileSnapshot = createFileSnapshot(otherFIle, otherProjectSnapshot); + + String hash = "a8998353e96320ec"; + DuplicationUnitDto duplicate = new DuplicationUnitDto() + .setHash(hash) + .setStartLine(40) + .setEndLine(55) + .setIndexInFile(0) + .setProjectSnapshotId(otherProjectSnapshot.getId()) + .setSnapshotId(otherFileSnapshot.getId()); + dbClient.duplicationDao().insert(dbSession, singletonList(duplicate)); + dbSession.commit(); + + BatchReport.CpdTextBlock originBlock = BatchReport.CpdTextBlock.newBuilder() + .setHash(hash) + .setStartLine(30) + .setEndLine(45) + .setStartTokenIndex(0) + .setEndTokenIndex(10) + .build(); + batchReportReader.putDuplicationBlocks(FILE_REF, asList(originBlock)); + + underTest.execute(); + + verifyZeroInteractions(integrateCrossProjectDuplications); + } + + @Test + public void nothing_to_do_when_no_cpd_text_blocks_found() throws Exception { + analysisMetadataHolder + .setCrossProjectDuplicationEnabled(true) + .setBranch(null) + .setBaseProjectSnapshot(baseProjectSnapshot); + + batchReportReader.putDuplicationBlocks(FILE_REF, Collections.emptyList()); + + underTest.execute(); + + verifyZeroInteractions(integrateCrossProjectDuplications); + } + + private ComponentDto createProject(String projectKey) { + ComponentDto project = ComponentTesting.newProjectDto().setKey(projectKey); + dbClient.componentDao().insert(dbSession, project); + dbSession.commit(); + return project; + } + + private SnapshotDto createProjectSnapshot(ComponentDto project) { + SnapshotDto projectSnapshot = SnapshotTesting.newSnapshotForProject(project); + dbClient.snapshotDao().insert(dbSession, projectSnapshot); + dbSession.commit(); + return projectSnapshot; + } + + private ComponentDto createFile(String fileKey, ComponentDto project) { + ComponentDto file = ComponentTesting.newFileDto(project) + .setKey(fileKey) + .setLanguage(XOO_LANGUAGE); + dbClient.componentDao().insert(dbSession, file); + dbSession.commit(); + return file; + } + + private SnapshotDto createFileSnapshot(ComponentDto file, SnapshotDto projectSnapshot) { + SnapshotDto fileSnapshot = SnapshotTesting.createForComponent(file, projectSnapshot); + dbClient.snapshotDao().insert(dbSession, fileSnapshot); + dbSession.commit(); + return fileSnapshot; + } + +} -- 2.39.5