package org.sonar.scanner.cpd;
import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.sonar.api.batch.fs.InputComponent;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.internal.DefaultInputComponent;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.duplications.block.Block;
private final InputComponentStore componentStore;
private final ProgressReport progressReport;
private final CpdSettings settings;
- private int count;
+ private final ExecutorService executorService;
+ private int count = 0;
private int total;
public CpdExecutor(CpdSettings settings, SonarCpdBlockIndex index, ReportPublisher publisher, InputComponentStore inputComponentCache) {
+ this(settings, index, publisher, inputComponentCache, Executors.newSingleThreadExecutor());
+ }
+
+ public CpdExecutor(CpdSettings settings, SonarCpdBlockIndex index, ReportPublisher publisher, InputComponentStore inputComponentCache, ExecutorService executorService) {
this.settings = settings;
this.index = index;
this.publisher = publisher;
this.componentStore = inputComponentCache;
this.progressReport = new ProgressReport("CPD computation", TimeUnit.SECONDS.toMillis(10));
+ this.executorService = executorService;
}
public void execute() {
@VisibleForTesting
void execute(long timeout) {
- total = index.noResources();
- int filesWithoutBlocks = index.noIndexedFiles() - total;
+ List<FileBlocks> components = new ArrayList<>(index.noResources());
+ Iterator<ResourceBlocks> it = index.iterator();
+
+ while (it.hasNext()) {
+ ResourceBlocks resourceBlocks = it.next();
+ Optional<FileBlocks> fileBlocks = toFileBlocks(resourceBlocks.resourceId(), resourceBlocks.blocks());
+ if (!fileBlocks.isPresent() || fileBlocks.get().getInputFile().status() == InputFile.Status.SAME) {
+ continue;
+ }
+ components.add(fileBlocks.get());
+ }
+
+ int filesWithoutBlocks = index.noIndexedFiles() - index.noResources();
if (filesWithoutBlocks > 0) {
LOG.info("{} {} had no CPD blocks", filesWithoutBlocks, pluralize(filesWithoutBlocks));
}
+
+ total = components.size();
progressReport.start(String.format("Calculating CPD for %d %s", total, pluralize(total)));
- ExecutorService executorService = Executors.newSingleThreadExecutor();
try {
- Iterator<ResourceBlocks> it = index.iterator();
-
- while (it.hasNext()) {
- ResourceBlocks resourceBlocks = it.next();
- runCpdAnalysis(executorService, resourceBlocks.resourceId(), resourceBlocks.blocks(), timeout);
+ for (FileBlocks fileBlocks : components) {
+ runCpdAnalysis(executorService, fileBlocks.getInputFile(), fileBlocks.getBlocks(), timeout);
count++;
}
progressReport.stop("CPD calculation finished");
}
@VisibleForTesting
- void runCpdAnalysis(ExecutorService executorService, String componentKey, final Collection<Block> fileBlocks, long timeout) {
- DefaultInputComponent component = (DefaultInputComponent) componentStore.getByKey(componentKey);
- if (component == null) {
- LOG.error("Resource not found in component store: {}. Skipping CPD computation for it", componentKey);
- return;
- }
-
- InputFile inputFile = (InputFile) component;
- if (inputFile.status() == InputFile.Status.SAME) {
- return;
- }
-
+ void runCpdAnalysis(ExecutorService executorService, DefaultInputFile inputFile, Collection<Block> fileBlocks, long timeout) {
LOG.debug("Detection of duplications for {}", inputFile.absolutePath());
progressReport.message(String.format("%d/%d - current file: %s", count, total, inputFile.absolutePath()));
filtered = duplications;
}
- saveDuplications(component, filtered);
+ saveDuplications(inputFile, filtered);
}
@VisibleForTesting final void saveDuplications(final DefaultInputComponent component, List<CloneGroup> duplications) {
publisher.getWriter().writeComponentDuplications(component.batchId(), reportDuplications);
}
+ private Optional<FileBlocks> toFileBlocks(String componentKey, Collection<Block> fileBlocks) {
+ DefaultInputFile component = (DefaultInputFile) componentStore.getByKey(componentKey);
+ if (component == null) {
+ LOG.error("Resource not found in component store: {}. Skipping CPD computation for it", componentKey);
+ return Optional.empty();
+ }
+ return Optional.of(new FileBlocks(component, fileBlocks));
+ }
+
private Duplication toReportDuplication(InputComponent component, Duplication.Builder dupBuilder, Duplicate.Builder blockBuilder, CloneGroup input) {
dupBuilder.clear();
ClonePart originBlock = input.getOriginPart();
}
return dupBuilder.build();
}
+
+ private static class FileBlocks {
+ public FileBlocks(DefaultInputFile inputFile, Collection<Block> blocks) {
+ this.inputFile = inputFile;
+ this.blocks = blocks;
+ }
+
+ private DefaultInputFile inputFile;
+ private Collection<Block> blocks;
+
+ public DefaultInputFile getInputFile() {
+ return inputFile;
+ }
+
+ public Collection<Block> getBlocks() {
+ return blocks;
+ }
+ }
}
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
import java.util.function.Consumer;
import javax.annotation.Nullable;
import org.junit.Before;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
+import org.mockito.ArgumentMatchers;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.batch.fs.internal.DefaultInputModule;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
public class CpdExecutorTest {
public ExpectedException thrown = ExpectedException.none();
private CpdExecutor executor;
+ private ExecutorService executorService = mock(ExecutorService.class);
private CpdSettings settings = mock(CpdSettings.class);
private ReportPublisher publisher = mock(ReportPublisher.class);
private SonarCpdBlockIndex index = new SonarCpdBlockIndex(publisher, settings);
DefaultInputModule inputModule = TestInputFileBuilder.newDefaultInputModule("foo", baseDir);
componentStore = new InputComponentStore(inputModule, mock(BranchConfiguration.class));
- executor = new CpdExecutor(settings, index, publisher, componentStore);
+ executor = new CpdExecutor(settings, index, publisher, componentStore, executorService);
reader = new ScannerReportReader(outputDir);
batchComponent1 = createComponent("src/Foo.php", 5);
batchComponent4 = createComponent("src/Foo4.php", 5, f -> f.setStatus(InputFile.Status.SAME));
}
- private DefaultInputFile createComponent(String relativePath, int lines) {
- return createComponent(relativePath, lines, f -> {
- });
- }
-
- private DefaultInputFile createComponent(String relativePath, int lines, Consumer<TestInputFileBuilder> config) {
- TestInputFileBuilder fileBuilder = new TestInputFileBuilder("foo", relativePath)
- .setModuleBaseDir(baseDir.toPath())
- .setLines(lines);
- config.accept(fileBuilder);
- DefaultInputFile file = fileBuilder.build();
- componentStore.put(file);
- return file;
- }
-
@Test
- public void testNothingToSave() {
+ public void dont_fail_if_nothing_to_save() {
executor.saveDuplications(batchComponent1, Collections.<CloneGroup>emptyList());
assertThat(reader.readComponentDuplications(batchComponent1.batchId())).hasSize(0);
}
@Test
- public void reportOneSimpleDuplicationBetweenTwoFiles() {
- List<CloneGroup> groups = Collections.singletonList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 2, 4),
+ public void should_save_single_duplication() {
+ List<CloneGroup> groups = Collections.singletonList(newCloneGroup(
+ new ClonePart(batchComponent1.key(), 0, 2, 4),
new ClonePart(batchComponent2.key(), 0, 15, 17)));
executor.saveDuplications(batchComponent1, groups);
}
@Test
- public void dontReportDuplicationOnUnmodifiedFileInSLB() {
- ExecutorService executorService = mock(ExecutorService.class);
- executor.runCpdAnalysis(executorService, batchComponent4.key(), Collections.emptyList(), 1000L);
-
- readDuplications(batchComponent4, 0);
- verifyZeroInteractions(executorService);
- }
-
- @Test
- public void reportDuplicationOnSameFile() {
- List<CloneGroup> groups = Collections.singletonList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204), new ClonePart(batchComponent1.key(), 0, 215, 414)));
+ public void should_save_duplication_on_same_file() {
+ List<CloneGroup> groups = Collections.singletonList(newCloneGroup(
+ new ClonePart(batchComponent1.key(), 0, 5, 204),
+ new ClonePart(batchComponent1.key(), 0, 215, 414)));
executor.saveDuplications(batchComponent1, groups);
Duplication[] dups = readDuplications(1);
}
@Test
- public void reportTooManyDuplicates() {
+ public void should_limit_number_of_references() {
// 1 origin part + 101 duplicates = 102
List<ClonePart> parts = new ArrayList<>(CpdExecutor.MAX_CLONE_PART_PER_GROUP + 2);
for (int i = 0; i < CpdExecutor.MAX_CLONE_PART_PER_GROUP + 2; i++) {
parts.add(new ClonePart(batchComponent1.key(), i, i, i + 1));
}
- List<CloneGroup> groups = Arrays.asList(CloneGroup.builder().setLength(0).setOrigin(parts.get(0)).setParts(parts).build());
+ List<CloneGroup> groups = Collections.singletonList(CloneGroup.builder().setLength(0).setOrigin(parts.get(0)).setParts(parts).build());
executor.saveDuplications(batchComponent1, groups);
Duplication[] dups = readDuplications(1);
}
@Test
- public void reportTooManyDuplications() throws Exception {
+ public void should_limit_number_of_clones() {
// 1 origin part + 101 duplicates = 102
List<CloneGroup> dups = new ArrayList<>(CpdExecutor.MAX_CLONE_GROUP_PER_FILE + 1);
for (int i = 0; i < CpdExecutor.MAX_CLONE_GROUP_PER_FILE + 1; i++) {
}
@Test
- public void reportOneDuplicatedGroupInvolvingMoreThanTwoFiles() throws Exception {
- List<CloneGroup> groups = Collections.singletonList(newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204),
- new ClonePart(batchComponent2.key(), 0, 15, 214), new ClonePart(batchComponent3.key(), 0, 25, 224)));
+ public void should_save_duplication_involving_three_files() {
+ List<CloneGroup> groups = Collections.singletonList(newCloneGroup(
+ new ClonePart(batchComponent1.key(), 0, 5, 204),
+ new ClonePart(batchComponent2.key(), 0, 15, 214),
+ new ClonePart(batchComponent3.key(), 0, 25, 224)));
executor.saveDuplications(batchComponent1, groups);
Duplication[] dups = readDuplications(1);
}
@Test
- public void reportTwoDuplicatedGroupsInvolvingThreeFiles() throws Exception {
+ public void should_save_two_duplicated_groups_involving_three_files() {
List<CloneGroup> groups = Arrays.asList(
newCloneGroup(new ClonePart(batchComponent1.key(), 0, 5, 204),
new ClonePart(batchComponent2.key(), 0, 15, 214)),
}
@Test
- public void failOnMissingComponent() {
- executor.runCpdAnalysis(null, "unknown", Collections.emptyList(), 1);
- readDuplications(0);
+ public void should_ignore_unmodified_files_in_SLB() {
+ Block block = Block.builder().setBlockHash(new ByteArray("AAAABBBBCCCC")).build();
+ index.insert(batchComponent4, Collections.singletonList(block));
+ executor.execute();
+
+ verify(executorService).shutdown();
+ verifyNoMoreInteractions(executorService);
+ readDuplications(batchComponent4, 0);
+ }
+
+ @Test
+ public void should_ignore_missing_component() {
+ Block block = Block.builder()
+ .setBlockHash(new ByteArray("AAAABBBBCCCC"))
+ .setResourceId("unknown")
+ .build();
+ index.insert(batchComponent1, Collections.singletonList(block));
+ executor.execute();
+
+ verify(executorService).shutdown();
+ verifyNoMoreInteractions(executorService);
+ readDuplications(batchComponent1, 0);
assertThat(logTester.logs(LoggerLevel.ERROR)).contains("Resource not found in component store: unknown. Skipping CPD computation for it");
}
@Test
- public void timeout() {
- for (int i = 1; i <= 2; i++) {
- DefaultInputFile component = createComponent("src/Foo" + i + ".php", 100);
- List<Block> blocks = new ArrayList<>();
- for (int j = 1; j <= 10000; j++) {
- blocks.add(Block.builder()
- .setResourceId(component.key())
- .setIndexInFile(j)
- .setLines(j, j + 1)
- .setUnit(j, j + 1)
- .setBlockHash(new ByteArray("abcd1234".getBytes()))
- .build());
- }
- index.insert(component, blocks);
- }
+ public void should_timeout() {
+ Block block = Block.builder()
+ .setBlockHash(new ByteArray("AAAABBBBCCCC"))
+ .setResourceId(batchComponent1.key())
+ .build();
+ index.insert(batchComponent1, Collections.singletonList(block));
+ when(executorService.submit(ArgumentMatchers.any(Callable.class))).thenReturn(new CompletableFuture());
executor.execute(1);
readDuplications(0);
assertThat(logTester.logs(LoggerLevel.WARN))
.usingElementComparator((l, r) -> l.matches(r) ? 0 : 1)
.containsOnly(
- "Timeout during detection of duplications for .*Foo1.php",
- "Timeout during detection of duplications for .*Foo2.php");
+ "Timeout during detection of duplications for .*Foo.php");
+ }
+
+ private DefaultInputFile createComponent(String relativePath, int lines) {
+ return createComponent(relativePath, lines, f -> {
+ });
+ }
+
+ private DefaultInputFile createComponent(String relativePath, int lines, Consumer<TestInputFileBuilder> config) {
+ TestInputFileBuilder fileBuilder = new TestInputFileBuilder("foo", relativePath)
+ .setModuleBaseDir(baseDir.toPath())
+ .setLines(lines);
+ config.accept(fileBuilder);
+ DefaultInputFile file = fileBuilder.build();
+ componentStore.put(file);
+ return file;
}
private Duplication[] readDuplications(int expected) {
projects / sonarqube.git / commitdiff