import org.sonar.server.computation.batch.TreeRootHolderRule;
import org.sonar.server.computation.component.Component;
import org.sonar.server.computation.component.ReportComponent;
+import org.sonar.server.computation.duplication.DuplicationRepositoryRule;
+import org.sonar.server.computation.duplication.TextBlock;
import org.sonar.server.computation.scm.ScmInfoRepositoryImpl;
import org.sonar.server.computation.source.SourceHashRepositoryImpl;
import org.sonar.server.computation.source.SourceLinesRepositoryImpl;
@Rule
public TemporaryFolder temp = new TemporaryFolder();
-
@Rule
public DbTester dbTester = DbTester.create(System2.INSTANCE);
-
@Rule
public Benchmark benchmark = new Benchmark();
-
@Rule
public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule();
-
@Rule
public MutableAnalysisMetadataHolderRule analysisMetadataHolder = new MutableAnalysisMetadataHolderRule();
+ @Rule
+ public DuplicationRepositoryRule duplicationRepository = DuplicationRepositoryRule.create(treeRootHolder);
@Test
public void benchmark() throws Exception {
SourceLinesRepositoryImpl sourceLinesRepository = new SourceLinesRepositoryImpl(batchReportReader);
SourceHashRepositoryImpl sourceHashRepository = new SourceHashRepositoryImpl(sourceLinesRepository);
ScmInfoRepositoryImpl scmInfoRepository = new ScmInfoRepositoryImpl(batchReportReader, analysisMetadataHolder, dbClient, sourceHashRepository);
- PersistFileSourcesStep step = new PersistFileSourcesStep(dbClient, System2.INSTANCE, treeRootHolder, batchReportReader, sourceLinesRepository, scmInfoRepository);
+ PersistFileSourcesStep step = new PersistFileSourcesStep(dbClient, System2.INSTANCE, treeRootHolder, batchReportReader, sourceLinesRepository, scmInfoRepository, duplicationRepository);
step.execute();
long end = System.currentTimeMillis();
long duration = end - start;
assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(NUMBER_OF_FILES);
- LOGGER.info(String.format("File sources has been persisted in %d ms", duration));
+ LOGGER.info(String.format("File sources have been persisted in %d ms", duration));
benchmark.expectAround("Duration to persist FILE_SOURCES", duration, 125000, Benchmark.DEFAULT_ERROR_MARGIN_PERCENTS);
}
List<Component> components = new ArrayList<>();
for (int fileRef = 2; fileRef <= NUMBER_OF_FILES + 1; fileRef++) {
- components.add(generateFileReport(writer, fileRef));
- project.addChildRef(fileRef);
+ ReportComponent component = ReportComponent.builder(Component.Type.FILE, fileRef).setUuid(Uuids.create()).setKey("PROJECT:" + fileRef).build();
+ components.add(component);
}
treeRootHolder.setRoot(ReportComponent.builder(Component.Type.PROJECT, 1)
.setUuid(PROJECT_UUID)
.setKey("PROJECT")
.addChildren(components.toArray(new Component[components.size()]))
.build());
+ for (int fileRef = 2; fileRef <= NUMBER_OF_FILES + 1; fileRef++) {
+ generateFileReport(writer, fileRef);
+ project.addChildRef(fileRef);
+ }
writer.writeComponent(project.build());
LineData lineData = new LineData();
for (int line = 1; line <= NUMBER_OF_LINES; line++) {
lineData.generateLineData(line);
+
+ duplicationRepository.addDuplication(fileRef, new TextBlock(line, line), new TextBlock(line + 1, line + 1));
+
}
writer.writeComponent(BatchReport.Component.newBuilder()
.setRef(fileRef)
writer.writeComponentChangesets(lineData.changesetsBuilder.setComponentRef(fileRef).build());
writer.writeComponentSyntaxHighlighting(fileRef, lineData.highlightings);
writer.writeComponentSymbols(fileRef, lineData.symbols);
- writer.writeComponentDuplications(fileRef, lineData.duplications);
return ReportComponent.builder(Component.Type.FILE, fileRef).setUuid(Uuids.create()).setKey("PROJECT:" + fileRef).build();
}
List<BatchReport.Coverage> coverages = new ArrayList<>();
List<BatchReport.SyntaxHighlighting> highlightings = new ArrayList<>();
List<BatchReport.Symbol> symbols = new ArrayList<>();
- List<BatchReport.Duplication> duplications = new ArrayList<>();
void generateLineData(int line) {
lines.add("line-" + line);
.setStartLine(line + 1).setEndLine(line + 1).setStartOffset(1).setEndOffset(3)
.build())
.build());
-
- duplications.add(BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(line)
- .setEndLine(line)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(line + 1)
- .setEndLine(line + 1)
- .build())
- .build())
- .build());
}
}
package org.sonar.server.computation.source;
-import java.io.Serializable;
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Ordering;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.sonar.batch.protocol.output.BatchReport;
+import java.util.Set;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import org.sonar.db.protobuf.DbFileSources;
+import org.sonar.server.computation.duplication.Duplication;
+import org.sonar.server.computation.duplication.InnerDuplicate;
+import org.sonar.server.computation.duplication.TextBlock;
-import static com.google.common.collect.Lists.newArrayList;
-import static com.google.common.collect.Maps.newHashMap;
+import static com.google.common.collect.FluentIterable.from;
public class DuplicationLineReader implements LineReader {
- private final List<BatchReport.Duplication> duplications;
- private final Map<BatchReport.TextRange, Integer> duplicationIdsByRange;
+ private final Map<TextBlock, Integer> duplicatedTextBlockIndexByTextBlock;
- public DuplicationLineReader(Iterator<BatchReport.Duplication> duplications) {
- this.duplications = newArrayList(duplications);
- // Sort duplication to have deterministic results and avoid false variation that would lead to an unnecessary update of the source files
- // data
- Collections.sort(this.duplications, new DuplicationComparator());
-
- this.duplicationIdsByRange = createDuplicationIdsByRange(this.duplications);
+ public DuplicationLineReader(Set<Duplication> duplications) {
+ this.duplicatedTextBlockIndexByTextBlock = createIndexOfDuplicatedTextBlocks(duplications);
}
@Override
public void read(DbFileSources.Line.Builder lineBuilder) {
- int line = lineBuilder.getLine();
- List<BatchReport.TextRange> blocks = findDuplicationBlockMatchingLine(line);
- for (BatchReport.TextRange block : blocks) {
- lineBuilder.addDuplication(duplicationIdsByRange.get(block));
+ Predicate<Map.Entry<TextBlock, Integer>> containsLine = new TextBlockContainsLine(lineBuilder.getLine());
+ for (Integer textBlockIndex : from(duplicatedTextBlockIndexByTextBlock.entrySet())
+ .filter(containsLine)
+ .transform(MapEntryToBlockId.INSTANCE)
+ // list is sorted to cope with the non-guaranteed order of Map entries which would trigger false detection of changes
+ // in {@link DbFileSources.Line#getDuplicationList()}
+ .toSortedList(Ordering.natural())) {
+ lineBuilder.addDuplication(textBlockIndex);
}
}
- private List<BatchReport.TextRange> findDuplicationBlockMatchingLine(int line) {
- List<BatchReport.TextRange> blocks = newArrayList();
- for (BatchReport.Duplication duplication : duplications) {
- if (matchLine(duplication.getOriginPosition(), line)) {
- blocks.add(duplication.getOriginPosition());
- }
- for (BatchReport.Duplicate duplicate : duplication.getDuplicateList()) {
- if (isDuplicationOnSameFile(duplicate) && matchLine(duplicate.getRange(), line)) {
- blocks.add(duplicate.getRange());
- }
- }
- }
- return blocks;
+ private static boolean isLineInBlock(TextBlock range, int line) {
+ return line >= range.getStart() && line <= range.getEnd();
}
- private static boolean isDuplicationOnSameFile(BatchReport.Duplicate duplicate) {
- return !duplicate.hasOtherFileRef();
+ /**
+ *
+ * <p>
+ * This method uses the natural order of TextBlocks to ensure that given the same set of TextBlocks, they get the same
+ * index. It avoids false detections of changes in {@link DbFileSources.Line#getDuplicationList()}.
+ * </p>
+ */
+ private static Map<TextBlock, Integer> createIndexOfDuplicatedTextBlocks(Collection<Duplication> duplications) {
+ List<TextBlock> duplicatedTextBlocks = extractAllDuplicatedTextBlocks(duplications);
+ Collections.sort(duplicatedTextBlocks);
+ return from(duplicatedTextBlocks)
+ .toMap(new TextBlockIndexGenerator());
}
- private static boolean matchLine(BatchReport.TextRange range, int line) {
- return range.getStartLine() <= line && line <= range.getEndLine();
+ /**
+ * Duplicated blocks in the current file are either {@link Duplication#getOriginal()} or {@link Duplication#getDuplicates()}
+ * when the {@link org.sonar.server.computation.duplication.Duplicate} is a {@link InnerDuplicate}.
+ * <p>
+ * The returned list is mutable on purpose because it will be sorted.
+ * </p>
+ *
+ * @see {@link #createIndexOfDuplicatedTextBlocks(Collection)}
+ */
+ private static List<TextBlock> extractAllDuplicatedTextBlocks(Collection<Duplication> duplications) {
+ List<TextBlock> duplicatedBlock = new ArrayList<>(duplications.size());
+ for (Duplication duplication : duplications) {
+ duplicatedBlock.add(duplication.getOriginal());
+ for (InnerDuplicate duplicate : from(duplication.getDuplicates()).filter(InnerDuplicate.class)) {
+ duplicatedBlock.add(duplicate.getTextBlock());
+ }
+ }
+ return duplicatedBlock;
}
- private static int length(BatchReport.TextRange range) {
- return (range.getEndLine() - range.getStartLine()) + 1;
- }
+ private static class TextBlockContainsLine implements Predicate<Map.Entry<TextBlock, Integer>> {
+ private final int line;
- private static Map<BatchReport.TextRange, Integer> createDuplicationIdsByRange(List<BatchReport.Duplication> duplications) {
- Map<BatchReport.TextRange, Integer> map = newHashMap();
- int blockId = 1;
- for (BatchReport.Duplication duplication : duplications) {
- map.put(duplication.getOriginPosition(), blockId);
- blockId++;
- for (BatchReport.Duplicate duplicate : duplication.getDuplicateList()) {
- if (isDuplicationOnSameFile(duplicate)) {
- map.put(duplicate.getRange(), blockId);
- blockId++;
- }
- }
+ public TextBlockContainsLine(int line) {
+ this.line = line;
+ }
+
+ @Override
+ public boolean apply(@Nonnull Map.Entry<TextBlock, Integer> input) {
+ return isLineInBlock(input.getKey(), line);
}
- return map;
}
- private static class DuplicationComparator implements Comparator<BatchReport.Duplication>, Serializable {
+ private enum MapEntryToBlockId implements Function<Map.Entry<TextBlock, Integer>, Integer> {
+ INSTANCE;
+
@Override
- public int compare(BatchReport.Duplication d1, BatchReport.Duplication d2) {
- if (d1.getOriginPosition().getStartLine() == d2.getOriginPosition().getStartLine()) {
- return Integer.compare(length(d1.getOriginPosition()), length(d2.getOriginPosition()));
- } else {
- return Integer.compare(d1.getOriginPosition().getStartLine(), d2.getOriginPosition().getStartLine());
- }
+ @Nonnull
+ public Integer apply(@Nonnull Map.Entry<TextBlock, Integer> input) {
+ return input.getValue();
}
}
+ private static class TextBlockIndexGenerator implements Function<TextBlock, Integer> {
+ int i = 1;
+
+ @Nullable
+ @Override
+ public Integer apply(TextBlock input) {
+ return i++;
+ }
+ }
}
@Override
public String getDescription() {
- return "Load inner and project duplications";
+ return "Load inner file and in project duplications";
}
@Override
package org.sonar.server.computation.step;
-import java.util.Iterator;
+import java.util.Set;
import org.apache.commons.lang.StringEscapeUtils;
import org.sonar.api.measures.CoreMetrics;
-import org.sonar.batch.protocol.output.BatchReport;
-import org.sonar.core.util.CloseableIterator;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.MyBatis;
import org.sonar.db.measure.MeasureDto;
import org.sonar.db.metric.MetricDto;
-import org.sonar.server.computation.batch.BatchReportReader;
import org.sonar.server.computation.component.Component;
import org.sonar.server.computation.component.CrawlerDepthLimit;
import org.sonar.server.computation.component.DbIdsRepository;
import org.sonar.server.computation.component.DepthTraversalTypeAwareCrawler;
import org.sonar.server.computation.component.ReportTreeRootHolder;
import org.sonar.server.computation.component.TypeAwareVisitorAdapter;
+import org.sonar.server.computation.duplication.CrossProjectDuplicate;
+import org.sonar.server.computation.duplication.Duplicate;
+import org.sonar.server.computation.duplication.Duplication;
+import org.sonar.server.computation.duplication.DuplicationRepository;
+import org.sonar.server.computation.duplication.InProjectDuplicate;
+import org.sonar.server.computation.duplication.InnerDuplicate;
+import org.sonar.server.computation.duplication.TextBlock;
import static org.sonar.server.computation.component.ComponentVisitor.Order.PRE_ORDER;
private final DbClient dbClient;
private final DbIdsRepository dbIdsRepository;
private final ReportTreeRootHolder treeRootHolder;
- private final BatchReportReader reportReader;
+ private final DuplicationRepository duplicationRepository;
- public PersistDuplicationsStep(DbClient dbClient, DbIdsRepository dbIdsRepository, ReportTreeRootHolder treeRootHolder, BatchReportReader reportReader) {
+ public PersistDuplicationsStep(DbClient dbClient, DbIdsRepository dbIdsRepository, ReportTreeRootHolder treeRootHolder,
+ DuplicationRepository duplicationRepository) {
this.dbClient = dbClient;
this.dbIdsRepository = dbIdsRepository;
this.treeRootHolder = treeRootHolder;
- this.reportReader = reportReader;
+ this.duplicationRepository = duplicationRepository;
}
@Override
try {
MetricDto duplicationMetric = dbClient.metricDao().selectOrFailByKey(session, CoreMetrics.DUPLICATIONS_DATA_KEY);
new DepthTraversalTypeAwareCrawler(new DuplicationVisitor(session, duplicationMetric))
- .visit(treeRootHolder.getRoot());
+ .visit(treeRootHolder.getRoot());
session.commit();
} finally {
MyBatis.closeQuietly(session);
@Override
public void visitFile(Component file) {
- visitComponent(file);
- }
-
- private void visitComponent(Component component) {
- try (CloseableIterator<BatchReport.Duplication> duplications = reportReader.readComponentDuplications(component.getReportAttributes().getRef())) {
- if (duplications.hasNext()) {
- saveDuplications(component, duplications);
- }
+ Set<Duplication> duplications = duplicationRepository.getDuplications(file);
+ if (!duplications.isEmpty()) {
+ saveDuplications(file, duplications);
}
}
- private void saveDuplications(Component component, Iterator<BatchReport.Duplication> duplications) {
+ private void saveDuplications(Component component, Iterable<Duplication> duplications) {
String duplicationXml = createXmlDuplications(component.getKey(), duplications);
MeasureDto measureDto = new MeasureDto()
- .setMetricId(duplicationMetric.getId())
- .setData(duplicationXml)
- .setComponentId(dbIdsRepository.getComponentId(component))
- .setSnapshotId(dbIdsRepository.getSnapshotId(component));
+ .setMetricId(duplicationMetric.getId())
+ .setData(duplicationXml)
+ .setComponentId(dbIdsRepository.getComponentId(component))
+ .setSnapshotId(dbIdsRepository.getSnapshotId(component));
dbClient.measureDao().insert(session, measureDto);
}
- private String createXmlDuplications(String componentKey, Iterator<BatchReport.Duplication> duplications) {
+ private String createXmlDuplications(String componentKey, Iterable<Duplication> duplications) {
StringBuilder xml = new StringBuilder();
xml.append("<duplications>");
- while (duplications.hasNext()) {
- BatchReport.Duplication duplication = duplications.next();
+ for (Duplication duplication : duplications) {
xml.append("<g>");
- appendDuplication(xml, componentKey, duplication.getOriginPosition());
- for (BatchReport.Duplicate duplicationBlock : duplication.getDuplicateList()) {
- processDuplicationBlock(xml, duplicationBlock, componentKey);
+ appendDuplication(xml, componentKey, duplication.getOriginal());
+ for (Duplicate duplicate : duplication.getDuplicates()) {
+ processDuplicationBlock(xml, duplicate, componentKey);
}
xml.append("</g>");
}
return xml.toString();
}
- private void processDuplicationBlock(StringBuilder xml, BatchReport.Duplicate duplicate, String componentKey) {
- if (duplicate.hasOtherFileRef()) {
- // Duplication is on a different file
- appendDuplication(xml, treeRootHolder.getComponentByRef(duplicate.getOtherFileRef()).getKey(), duplicate);
- } else {
+ private void processDuplicationBlock(StringBuilder xml, Duplicate duplicate, String componentKey) {
+ if (duplicate instanceof InnerDuplicate) {
// Duplication is on a the same file
appendDuplication(xml, componentKey, duplicate);
+ } else if (duplicate instanceof InProjectDuplicate) {
+ // Duplication is on a different file
+ appendDuplication(xml, ((InProjectDuplicate) duplicate).getFile().getKey(), duplicate);
+ } else if (duplicate instanceof CrossProjectDuplicate) {
+ // componentKey is only set for cross project duplications
+ String crossProjectComponentKey = ((CrossProjectDuplicate) duplicate).getFileKey();
+ appendDuplication(xml, crossProjectComponentKey, duplicate);
+ } else {
+ throw new IllegalArgumentException("Unsupported type of Duplicate " + duplicate.getClass().getName());
}
}
- private void appendDuplication(StringBuilder xml, String componentKey, BatchReport.Duplicate duplicate) {
- appendDuplication(xml, componentKey, duplicate.getRange());
+ private void appendDuplication(StringBuilder xml, String componentKey, Duplicate duplicate) {
+ appendDuplication(xml, componentKey, duplicate.getTextBlock());
}
- private void appendDuplication(StringBuilder xml, String componentKey, BatchReport.TextRange range) {
- int length = range.getEndLine() - range.getStartLine() + 1;
- xml.append("<b s=\"").append(range.getStartLine())
- .append("\" l=\"").append(length)
- .append("\" r=\"").append(StringEscapeUtils.escapeXml(componentKey))
- .append("\"/>");
+ private void appendDuplication(StringBuilder xml, String componentKey, TextBlock textBlock) {
+ int length = textBlock.getEnd() - textBlock.getStart() + 1;
+ xml.append("<b s=\"").append(textBlock.getStart())
+ .append("\" l=\"").append(length)
+ .append("\" r=\"").append(StringEscapeUtils.escapeXml(componentKey))
+ .append("\"/>");
}
}
import org.sonar.server.computation.component.DepthTraversalTypeAwareCrawler;
import org.sonar.server.computation.component.TreeRootHolder;
import org.sonar.server.computation.component.TypeAwareVisitorAdapter;
+import org.sonar.server.computation.duplication.DuplicationRepository;
import org.sonar.server.computation.scm.Changeset;
import org.sonar.server.computation.scm.ScmInfo;
import org.sonar.server.computation.scm.ScmInfoRepository;
private final BatchReportReader reportReader;
private final SourceLinesRepository sourceLinesRepository;
private final ScmInfoRepository scmInfoRepository;
+ private final DuplicationRepository duplicationRepository;
public PersistFileSourcesStep(DbClient dbClient, System2 system2, TreeRootHolder treeRootHolder, BatchReportReader reportReader, SourceLinesRepository sourceLinesRepository,
- ScmInfoRepository scmInfoRepository) {
+ ScmInfoRepository scmInfoRepository, DuplicationRepository duplicationRepository) {
this.dbClient = dbClient;
this.system2 = system2;
this.treeRootHolder = treeRootHolder;
this.reportReader = reportReader;
this.sourceLinesRepository = sourceLinesRepository;
this.scmInfoRepository = scmInfoRepository;
+ this.duplicationRepository = duplicationRepository;
}
@Override
int fileRef = file.getReportAttributes().getRef();
BatchReport.Component component = reportReader.readComponent(fileRef);
CloseableIterator<String> linesIterator = sourceLinesRepository.readLines(file);
- LineReaders lineReaders = new LineReaders(reportReader, scmInfoRepository, file);
+ LineReaders lineReaders = new LineReaders(reportReader, scmInfoRepository, duplicationRepository, file);
try {
ComputeFileSourceData computeFileSourceData = new ComputeFileSourceData(linesIterator, lineReaders.readers(), component.getLines());
ComputeFileSourceData.Data fileSourceData = computeFileSourceData.compute();
@CheckForNull
private final ScmLineReader scmLineReader;
- LineReaders(BatchReportReader reportReader, ScmInfoRepository scmInfoRepository, Component component) {
+ LineReaders(BatchReportReader reportReader, ScmInfoRepository scmInfoRepository, DuplicationRepository duplicationRepository, Component component) {
int componentRef = component.getReportAttributes().getRef();
CloseableIterator<BatchReport.Coverage> coverageIt = reportReader.readComponentCoverage(componentRef);
closeables.add(coverageIt);
closeables.add(symbolsIt);
readers.add(new SymbolsLineReader(component, symbolsIt, rangeOffsetConverter));
- CloseableIterator<BatchReport.Duplication> duplicationsIt = reportReader.readComponentDuplications(componentRef);
- closeables.add(duplicationsIt);
- readers.add(new DuplicationLineReader(duplicationsIt));
+ readers.add(new DuplicationLineReader(duplicationRepository.getDuplications(component)));
}
List<LineReader> readers() {
return closeableIterator(this.duplications.get(componentRef));
}
- public void putDuplications(int componentRef, List<BatchReport.Duplication> duplications) {
- this.duplications.put(componentRef, duplications);
+ public void putDuplications(int componentRef, BatchReport.Duplication... duplications) {
+ this.duplications.put(componentRef, Arrays.asList(duplications));
}
@Override
package org.sonar.server.computation.source;
-import com.google.common.collect.Iterators;
+import com.google.common.collect.ImmutableSet;
+import java.util.Arrays;
+import java.util.Collections;
import org.junit.Test;
-import org.sonar.batch.protocol.output.BatchReport;
-import org.sonar.core.util.CloseableIterator;
import org.sonar.db.protobuf.DbFileSources;
+import org.sonar.server.computation.component.Component;
+import org.sonar.server.computation.component.ReportComponent;
+import org.sonar.server.computation.duplication.CrossProjectDuplicate;
+import org.sonar.server.computation.duplication.Duplicate;
+import org.sonar.server.computation.duplication.Duplication;
+import org.sonar.server.computation.duplication.InProjectDuplicate;
+import org.sonar.server.computation.duplication.InnerDuplicate;
+import org.sonar.server.computation.duplication.TextBlock;
import static org.assertj.core.api.Assertions.assertThat;
@Test
public void read_nothing() {
- DuplicationLineReader reader = new DuplicationLineReader(CloseableIterator.<BatchReport.Duplication>emptyCloseableIterator());
+ DuplicationLineReader reader = new DuplicationLineReader(Collections.<Duplication>emptySet());
reader.read(line1);
@Test
public void read_duplication_with_duplicates_on_same_file() {
- DuplicationLineReader reader = new DuplicationLineReader(Iterators.forArray(
- BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(2)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(3)
- .setEndLine(4)
- .build())
- .build())
- .build()));
+ DuplicationLineReader reader = duplicationLineReader(duplication(1, 2, innerDuplicate(3, 4)));
reader.read(line1);
reader.read(line2);
@Test
public void read_duplication_with_duplicates_on_other_file() {
- DuplicationLineReader reader = new DuplicationLineReader(Iterators.forArray(
- BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(2)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setOtherFileRef(2)
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(3)
- .setEndLine(4)
- .build())
- .build())
- .build()));
+ DuplicationLineReader reader = duplicationLineReader(
+ duplication(
+ 1, 2,
+ new InProjectDuplicate(fileComponent(1).build(), new TextBlock(3, 4))));
+
+ reader.read(line1);
+ reader.read(line2);
+ reader.read(line3);
+ reader.read(line4);
+
+ assertThat(line1.getDuplicationList()).containsExactly(1);
+ assertThat(line2.getDuplicationList()).containsExactly(1);
+ assertThat(line3.getDuplicationList()).isEmpty();
+ assertThat(line4.getDuplicationList()).isEmpty();
+ }
+
+ @Test
+ public void read_duplication_with_duplicates_on_other_file_from_other_project() {
+ DuplicationLineReader reader = duplicationLineReader(
+ duplication(
+ 1, 2,
+ new CrossProjectDuplicate("other-component-key-from-another-project", new TextBlock(3, 4))));
reader.read(line1);
reader.read(line2);
@Test
public void read_many_duplications() {
- DuplicationLineReader reader = new DuplicationLineReader(Iterators.forArray(
- BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(1)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(2)
- .setEndLine(2)
- .build())
- .build())
- .build(),
- BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(2)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(3)
- .setEndLine(4)
- .build())
- .build())
- .build()));
+ DuplicationLineReader reader = duplicationLineReader(
+ duplication(
+ 1, 1,
+ innerDuplicate(2, 2)),
+ duplication(
+ 1, 2,
+ innerDuplicate(3, 4))
+ );
reader.read(line1);
reader.read(line2);
reader.read(line3);
reader.read(line4);
- assertThat(line1.getDuplicationList()).containsExactly(1, 3);
+ assertThat(line1.getDuplicationList()).containsExactly(1, 2);
assertThat(line2.getDuplicationList()).containsExactly(2, 3);
assertThat(line3.getDuplicationList()).containsExactly(4);
assertThat(line4.getDuplicationList()).containsExactly(4);
@Test
public void should_be_sorted_by_line_block() {
- DuplicationLineReader reader = new DuplicationLineReader(Iterators.forArray(
- BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(2)
- .setEndLine(2)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(4)
- .setEndLine(4)
- .build())
- .build())
- .build(),
- BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(1)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(3)
- .setEndLine(3)
- .build())
- .build())
- .build()));
+ DuplicationLineReader reader = duplicationLineReader(
+ duplication(
+ 2, 2,
+ innerDuplicate(4, 4)),
+ duplication(
+ 1, 1,
+ innerDuplicate(3, 3))
+ );
reader.read(line1);
reader.read(line2);
reader.read(line4);
assertThat(line1.getDuplicationList()).containsExactly(1);
- assertThat(line2.getDuplicationList()).containsExactly(3);
- assertThat(line3.getDuplicationList()).containsExactly(2);
+ assertThat(line2.getDuplicationList()).containsExactly(2);
+ assertThat(line3.getDuplicationList()).containsExactly(3);
assertThat(line4.getDuplicationList()).containsExactly(4);
}
@Test
public void should_be_sorted_by_line_length() {
- DuplicationLineReader reader = new DuplicationLineReader(Iterators.forArray(
- BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(2)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(3)
- .setEndLine(4)
- .build())
- .build())
- .build(),
- BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(1)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(4)
- .setEndLine(4)
- .build())
- .build())
- .build()));
+ DuplicationLineReader reader = duplicationLineReader(
+ duplication(
+ 1, 2,
+ innerDuplicate(3, 4)),
+ duplication(
+ 1, 1,
+ innerDuplicate(4, 4)
+ )
+ );
reader.read(line1);
reader.read(line2);
reader.read(line3);
reader.read(line4);
- assertThat(line1.getDuplicationList()).containsExactly(1, 3);
- assertThat(line2.getDuplicationList()).containsExactly(3);
- assertThat(line3.getDuplicationList()).containsExactly(4);
- assertThat(line4.getDuplicationList()).containsExactly(2, 4);
+ assertThat(line1.getDuplicationList()).containsExactly(1, 2);
+ assertThat(line2.getDuplicationList()).containsExactly(2);
+ assertThat(line3.getDuplicationList()).containsExactly(3);
+ assertThat(line4.getDuplicationList()).containsExactly(3, 4);
+ }
+
+ private static ReportComponent.Builder fileComponent(int ref) {
+ return ReportComponent.builder(Component.Type.FILE, ref);
+ }
+
+ private static DuplicationLineReader duplicationLineReader(Duplication... duplications) {
+ return new DuplicationLineReader(ImmutableSet.copyOf(Arrays.asList(duplications)));
+ }
+
+ private static Duplication duplication(int originalStart, int originalEnd, Duplicate... duplicates) {
+ return new Duplication(new TextBlock(originalStart, originalEnd), Arrays.asList(duplicates));
+ }
+
+ private static InnerDuplicate innerDuplicate(int start, int end) {
+ return new InnerDuplicate(new TextBlock(start, end));
}
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.computation.step;
+
+import java.util.Arrays;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.server.computation.batch.BatchReportReaderRule;
+import org.sonar.server.computation.batch.TreeRootHolderRule;
+import org.sonar.server.computation.component.Component;
+import org.sonar.server.computation.duplication.Duplicate;
+import org.sonar.server.computation.duplication.Duplication;
+import org.sonar.server.computation.duplication.DuplicationRepositoryRule;
+import org.sonar.server.computation.duplication.InProjectDuplicate;
+import org.sonar.server.computation.duplication.InnerDuplicate;
+import org.sonar.server.computation.duplication.TextBlock;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.sonar.server.computation.component.Component.Type.FILE;
+import static org.sonar.server.computation.component.Component.Type.PROJECT;
+import static org.sonar.server.computation.component.ReportComponent.builder;
+
+public class LoadDuplicationsFromReportStepTest {
+ private static final int LINE = 2;
+ private static final int OTHER_LINE = 300;
+ private static final int ROOT_REF = 1;
+ private static final int FILE_1_REF = 11;
+ private static final int FILE_2_REF = 12;
+
+ @Rule
+ public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule().setRoot(
+ builder(PROJECT, ROOT_REF)
+ .addChildren(
+ builder(FILE, FILE_1_REF).build(),
+ builder(FILE, FILE_2_REF).build()
+ )
+ .build()
+ );
+ @Rule
+ public BatchReportReaderRule reportReader = new BatchReportReaderRule();
+ @Rule
+ public DuplicationRepositoryRule duplicationRepository = DuplicationRepositoryRule.create(treeRootHolder);
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ private LoadDuplicationsFromReportStep underTest = new LoadDuplicationsFromReportStep(treeRootHolder, reportReader, duplicationRepository);
+
+ @Test
+ public void verify_description() {
+ assertThat(underTest.getDescription()).isEqualTo("Load inner file and in project duplications");
+ }
+
+ @Test
+ public void loads_no_duplications_if_reader_has_no_duplication() {
+ underTest.execute();
+
+ assertNoDuplication(FILE_1_REF);
+ }
+
+ @Test
+ public void loads_duplication_without_otherFileRef_as_inner_duplication() {
+ reportReader.putDuplications(FILE_2_REF, createDuplication(singleLineTextRange(LINE), createInnerDuplicate(LINE + 1)));
+
+ underTest.execute();
+
+ assertNoDuplication(FILE_1_REF);
+ assertDuplications(FILE_2_REF, singleLineTextBlock(LINE), new InnerDuplicate(singleLineTextBlock(LINE + 1)));
+ }
+
+ @Test
+ public void loads_duplication_with_otherFileRef_as_inProject_duplication() {
+ reportReader.putDuplications(FILE_1_REF, createDuplication(singleLineTextRange(LINE), createInProjectDuplicate(FILE_2_REF, LINE + 1)));
+
+ underTest.execute();
+
+ assertDuplications(FILE_1_REF, singleLineTextBlock(LINE), new InProjectDuplicate(treeRootHolder.getComponentByRef(FILE_2_REF), singleLineTextBlock(LINE + 1)));
+ assertNoDuplication(FILE_2_REF);
+ }
+
+ @Test
+ public void loads_multiple_duplications_with_multiple_duplicates() {
+ reportReader.putDuplications(
+ FILE_2_REF,
+ createDuplication(
+ singleLineTextRange(LINE),
+ createInnerDuplicate(LINE + 1), createInnerDuplicate(LINE + 2), createInProjectDuplicate(FILE_1_REF, LINE), createInProjectDuplicate(FILE_1_REF, LINE + 10)),
+ createDuplication(
+ singleLineTextRange(OTHER_LINE),
+ createInProjectDuplicate(FILE_1_REF, OTHER_LINE)),
+ createDuplication(
+ singleLineTextRange(OTHER_LINE + 80),
+ createInnerDuplicate(LINE), createInnerDuplicate(LINE + 10))
+ );
+
+ underTest.execute();
+
+ Component file1Component = treeRootHolder.getComponentByRef(FILE_1_REF);
+ assertThat(duplicationRepository.getDuplications(FILE_2_REF)).containsOnly(
+ duplication(
+ singleLineTextBlock(LINE),
+ new InnerDuplicate(singleLineTextBlock(LINE + 1)), new InnerDuplicate(singleLineTextBlock(LINE + 2)), new InProjectDuplicate(file1Component, singleLineTextBlock(LINE)),
+ new InProjectDuplicate(file1Component, singleLineTextBlock(LINE + 10))),
+ duplication(
+ singleLineTextBlock(OTHER_LINE),
+ new InProjectDuplicate(file1Component, singleLineTextBlock(OTHER_LINE))
+ ),
+ duplication(
+ singleLineTextBlock(OTHER_LINE + 80),
+ new InnerDuplicate(singleLineTextBlock(LINE)), new InnerDuplicate(singleLineTextBlock(LINE + 10))
+ )
+ );
+ }
+
+ @Test
+ public void loads_duplication_with_otherFileRef_throws_IAE_if_component_does_not_exist() {
+ int line = 2;
+ reportReader.putDuplications(FILE_1_REF, createDuplication(singleLineTextRange(line), createInProjectDuplicate(666, line + 1)));
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Component with ref '666' hasn't been found");
+
+ underTest.execute();
+ }
+
+ @Test
+ public void loads_duplication_with_otherFileRef_throws_IAE_if_references_itself() {
+ int line = 2;
+ reportReader.putDuplications(FILE_1_REF, createDuplication(singleLineTextRange(line), createInProjectDuplicate(FILE_1_REF, line + 1)));
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("file and otherFile Components can not be the same");
+
+ underTest.execute();
+ }
+
+ private void assertDuplications(int fileRef, TextBlock original, Duplicate... duplicates) {
+ assertThat(duplicationRepository.getDuplications(fileRef)).containsExactly(duplication(original, duplicates));
+ }
+
+ private static Duplication duplication(TextBlock original, Duplicate... duplicates) {
+ return new Duplication(original, Arrays.asList(duplicates));
+ }
+
+ private TextBlock singleLineTextBlock(int line) {
+ return new TextBlock(line, line);
+ }
+
+ private static BatchReport.Duplication createDuplication(BatchReport.TextRange original, BatchReport.Duplicate... duplicates) {
+ BatchReport.Duplication.Builder builder = BatchReport.Duplication.newBuilder()
+ .setOriginPosition(original);
+ for (BatchReport.Duplicate duplicate : duplicates) {
+ builder.addDuplicate(duplicate);
+ }
+ return builder.build();
+ }
+
+ private static BatchReport.Duplicate createInnerDuplicate(int line) {
+ return BatchReport.Duplicate.newBuilder()
+ .setRange(singleLineTextRange(line))
+ .build();
+ }
+
+ private static BatchReport.Duplicate createInProjectDuplicate(int componentRef, int line) {
+ return BatchReport.Duplicate.newBuilder()
+ .setOtherFileRef(componentRef)
+ .setRange(singleLineTextRange(line))
+ .build();
+ }
+
+ private static BatchReport.TextRange singleLineTextRange(int line) {
+ return BatchReport.TextRange.newBuilder()
+ .setStartLine(line)
+ .setEndLine(line)
+ .build();
+ }
+
+ private void assertNoDuplication(int fileRef) {
+ assertThat(duplicationRepository.getDuplications(fileRef)).isEmpty();
+ }
+}
import org.junit.experimental.categories.Category;
import org.sonar.api.measures.CoreMetrics;
import org.sonar.api.utils.System2;
-import org.sonar.batch.protocol.output.BatchReport;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;
import org.sonar.db.metric.MetricDto;
-import org.sonar.server.computation.batch.BatchReportReaderRule;
import org.sonar.server.computation.batch.TreeRootHolderRule;
import org.sonar.server.computation.component.Component;
import org.sonar.server.computation.component.DbIdsRepositoryImpl;
import org.sonar.server.computation.component.ReportComponent;
+import org.sonar.server.computation.duplication.DuplicationRepositoryRule;
+import org.sonar.server.computation.duplication.TextBlock;
import org.sonar.test.DbTests;
-import static com.google.common.collect.Lists.newArrayList;
import static org.assertj.core.api.Assertions.assertThat;
@Category(DbTests.class)
@Rule
public DbTester dbTester = DbTester.create(System2.INSTANCE);
@Rule
- public BatchReportReaderRule reportReader = new BatchReportReaderRule();
+ public DuplicationRepositoryRule duplicationRepository = DuplicationRepositoryRule.create(treeRootHolder);
DbIdsRepositoryImpl dbIdsRepository = new DbIdsRepositoryImpl();
-
DbSession session = dbTester.getSession();
-
DbClient dbClient = dbTester.getDbClient();
PersistDuplicationsStep underTest;
@Before
public void setup() {
dbTester.truncateTables();
- underTest = new PersistDuplicationsStep(dbClient, dbIdsRepository, treeRootHolder, reportReader);
+ underTest = new PersistDuplicationsStep(dbClient, dbIdsRepository, treeRootHolder, duplicationRepository);
}
@Override
MetricDto duplicationMetric = saveDuplicationMetric();
initReportWithProjectAndFile();
- BatchReport.Duplication duplication = BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(5)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(6)
- .setEndLine(10)
- .build())
- .build())
- .build();
- reportReader.putDuplications(FILE_1_REF, newArrayList(duplication));
+ duplicationRepository.addDuplication(FILE_1_REF, new TextBlock(1, 5), new TextBlock(6, 10));
underTest.execute();
saveDuplicationMetric();
initReportWithProjectAndFile();
- BatchReport.Duplication duplication = BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(5)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setOtherFileRef(FILE_2_REF).setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(6)
- .setEndLine(10)
- .build())
- .build())
- .build();
- reportReader.putDuplications(FILE_1_REF, newArrayList(duplication));
+ duplicationRepository.addDuplication(FILE_1_REF, new TextBlock(1, 5), 3, new TextBlock(6, 10));
underTest.execute();
assertThat(dto.get("textValue")).isEqualTo("<duplications><g><b s=\"1\" l=\"5\" r=\"PROJECT_KEY:file\"/><b s=\"6\" l=\"5\" r=\"PROJECT_KEY:file2\"/></g></duplications>");
}
+ @Test
+ public void persist_duplications_on_different_projects() {
+ saveDuplicationMetric();
+ initReportWithProjectAndFile();
+
+ duplicationRepository.addDuplication(FILE_1_REF, new TextBlock(1, 5), "PROJECT2_KEY:file2", new TextBlock(6, 10));
+
+ underTest.execute();
+
+ assertThat(dbTester.countRowsOfTable("project_measures")).isEqualTo(1);
+
+ Map<String, Object> dto = dbTester.selectFirst("select snapshot_id as \"snapshotId\", text_value as \"textValue\" from project_measures");
+ assertThat(dto.get("snapshotId")).isEqualTo(11L);
+ assertThat(dto.get("textValue")).isEqualTo("<duplications><g><b s=\"1\" l=\"5\" r=\"PROJECT_KEY:file\"/><b s=\"6\" l=\"5\" r=\"PROJECT2_KEY:file2\"/></g></duplications>");
+ }
+
private void initReportWithProjectAndFile() {
Component file1 = ReportComponent.builder(Component.Type.FILE, FILE_1_REF).setUuid("BCDE").setKey("PROJECT_KEY:file").build();
Component file2 = ReportComponent.builder(Component.Type.FILE, FILE_2_REF).setUuid("CDEF").setKey("PROJECT_KEY:file2").build();
import org.sonar.server.computation.batch.TreeRootHolderRule;
import org.sonar.server.computation.component.Component;
import org.sonar.server.computation.component.ReportComponent;
+import org.sonar.server.computation.duplication.DuplicationRepositoryRule;
+import org.sonar.server.computation.duplication.TextBlock;
import org.sonar.server.computation.scm.Changeset;
import org.sonar.server.computation.scm.ScmInfoRepositoryRule;
import org.sonar.server.computation.source.SourceLinesRepositoryRule;
public ScmInfoRepositoryRule scmInfoRepository = new ScmInfoRepositoryRule();
@Rule
public SourceLinesRepositoryRule fileSourceRepository = new SourceLinesRepositoryRule();
+ @Rule
+ public DuplicationRepositoryRule duplicationRepository = DuplicationRepositoryRule.create(treeRootHolder);
private DbClient dbClient = dbTester.getDbClient();
private DbSession session = dbTester.getSession();
public void setup() {
dbTester.truncateTables();
when(system2.now()).thenReturn(NOW);
- underTest = new PersistFileSourcesStep(dbClient, system2, treeRootHolder, reportReader, fileSourceRepository, scmInfoRepository);
+ underTest = new PersistFileSourcesStep(dbClient, system2, treeRootHolder, reportReader, fileSourceRepository, scmInfoRepository, duplicationRepository);
}
@Override
public void persist_duplication() {
initBasicReport(1);
- reportReader.putDuplications(FILE_REF, newArrayList(
- BatchReport.Duplication.newBuilder()
- .setOriginPosition(BatchReport.TextRange.newBuilder()
- .setStartLine(1)
- .setEndLine(2)
- .build())
- .addDuplicate(BatchReport.Duplicate.newBuilder()
- .setRange(BatchReport.TextRange.newBuilder()
- .setStartLine(3)
- .setEndLine(4)
- .build())
- .build())
- .build()));
+ duplicationRepository.addDuplication(FILE_REF, new TextBlock(1, 2), new TextBlock(3, 4));
underTest.execute();