From: Janos Gyerik Date: Mon, 10 Sep 2018 15:40:51 +0000 (+0200) Subject: SONAR-11125 Build a pruned component tree without unchanged files for selected visitors X-Git-Tag: 7.5~468 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=3e857050df9d80e3e1e2cf703f4b30d85489590b;p=sonarqube.git SONAR-11125 Build a pruned component tree without unchanged files for selected visitors --- diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/Component.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/Component.java index af7a00e2ed5..e76ee80f181 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/Component.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/Component.java @@ -58,6 +58,10 @@ public interface Component { } } + /** + * On a long-living branch, CHANGED and ADDED are relative to the previous analysis. + * On a short-living branch and pull request, these are relative to the base branch. + */ enum Status { UNAVAILABLE, SAME, CHANGED, ADDED } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java index 427bb87bd52..bc605c43db2 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java @@ -20,6 +20,7 @@ package org.sonar.ce.task.projectanalysis.component; import java.util.List; +import java.util.Objects; import java.util.function.Function; import javax.annotation.CheckForNull; import javax.annotation.Nullable; @@ -145,6 +146,74 @@ public class ComponentTreeBuilder { } } + public Component buildChangedComponentTreeRoot(Component project) { + return buildChangedComponentTree(project); + } + + private static ComponentImpl.Builder changedComponentBuilder(Component component) { + return ComponentImpl.builder(component.getType()) + .setUuid(component.getUuid()) + .setKey(component.getKey()) + .setPublicKey(component.getPublicKey()) + .setStatus(component.getStatus()) + .setReportAttributes(component.getReportAttributes()) + .setName(component.getName()) + .setDescription(component.getDescription()); + } + + @Nullable + private static Component buildChangedComponentTree(Component component) { + switch (component.getType()) { + case PROJECT: + return buildChangedProject(component); + + case MODULE: + case DIRECTORY: + return buildChangedIntermediate(component); + + case FILE: + return buildChangedFile(component); + + default: + throw new IllegalArgumentException(format("Unsupported component type '%s'", component.getType())); + } + } + + private static Component buildChangedProject(Component component) { + return changedComponentBuilder(component) + .addChildren(buildChangedComponentChildren(component)) + .build(); + } + + @Nullable + private static Component buildChangedIntermediate(Component component) { + List children = buildChangedComponentChildren(component); + if (children.isEmpty()) { + return null; + } + return changedComponentBuilder(component) + .addChildren(children) + .build(); + } + + @Nullable + private static Component buildChangedFile(Component component) { + if (component.getStatus() == Component.Status.SAME) { + return null; + } + return changedComponentBuilder(component) + .setFileAttributes(component.getFileAttributes()) + .build(); + } + + private static List buildChangedComponentChildren(Component component) { + return component.getChildren() + .stream() + .map(ComponentTreeBuilder::buildChangedComponentTree) + .filter(Objects::nonNull) + .collect(MoreCollectors.toList()); + } + private void setNameAndDescription(ScannerReport.Component component, ComponentImpl.Builder builder) { if (branch.isMain()) { builder diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolder.java index 880286b1161..e63bf33c176 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolder.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolder.java @@ -22,13 +22,18 @@ package org.sonar.ce.task.projectanalysis.component; public interface MutableTreeRootHolder extends TreeRootHolder { /** - * Sets the root of the component tree in the TreeRootHolder. Settings a root more than once is allowed but it can - * never be set to {@code null}. + * Sets the root of the component tree in the TreeRootHolder. * - * @param newRoot a {@link Component}, can not be {@code null} - * * @throws NullPointerException if {@code newRoot} is {@code null} * @throws IllegalStateException if root {@link Component} has already been set */ MutableTreeRootHolder setRoot(Component newRoot); + + /** + * Sets the root of the components that were in the scanner report in the TreeRootHolder. + * + * @throws NullPointerException if {@code newRoot} is {@code null} + * @throws IllegalStateException if extended tree root {@link Component} has already been set + */ + MutableTreeRootHolder setReportTreeRoot(Component newRoot); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolder.java index 85785ed6332..347c7facad6 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolder.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolder.java @@ -29,10 +29,25 @@ public interface TreeRootHolder { * The root of the tree, for example the project or the portfolio. * With branches, it will refer to the root component of the branch. * + * On a short living branch or pull request, it contains ONLY: + * 1. The PROJECT component (tree root) + * 2. The FILE components whose status is not SAME + * 3. Intermediary MODULE and DIRECTORY components that lead to FILE leafs that are not SAME + * * @throws IllegalStateException if the holder is empty (ie. there is no root yet) */ Component getRoot(); + /** + * The root of the components that were in the scanner report. + * This tree may include components that are not persisted, + * just kept in memory for computation purposes, such as overall coverage + * in short living branches or pull requests. + * + * @throws IllegalStateException if the holder is empty (ie. there is no root yet) + */ + Component getReportTreeRoot(); + /** * Return a component by its batch reference * diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderImpl.java index 9598d93aaba..91f47b639f1 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderImpl.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderImpl.java @@ -36,6 +36,7 @@ public class TreeRootHolderImpl implements MutableTreeRootHolder { private Map componentsByRef; private Component root; + private Component extendedTreeRoot; @Override public MutableTreeRootHolder setRoot(Component root) { @@ -50,6 +51,19 @@ public class TreeRootHolderImpl implements MutableTreeRootHolder { return this.root; } + @Override + public MutableTreeRootHolder setReportTreeRoot(Component root) { + checkState(this.extendedTreeRoot == null, "extended tree root can not be set twice in holder"); + this.extendedTreeRoot = requireNonNull(root, "extended tree root can not be null"); + return this; + } + + @Override + public Component getReportTreeRoot() { + checkInitialized(); + return this.extendedTreeRoot; + } + @Override public Component getComponentByRef(int ref) { return getOptionalComponentByRef(ref) diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java index fee73596d14..206f7d8a4c8 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java @@ -84,9 +84,17 @@ public class BuildComponentTreeStep implements ComputationStep { analysisMetadataHolder.getBranch(), baseAnalysis); String relativePathFromScmRoot = reportReader.readMetadata().getRelativePathFromScmRoot(); - Component project = builder.buildProject(reportProject, relativePathFromScmRoot); - treeRootHolder.setRoot(project); + Component reportTreeRoot = builder.buildProject(reportProject, relativePathFromScmRoot); + treeRootHolder.setReportTreeRoot(reportTreeRoot); + + if (analysisMetadataHolder.isShortLivingBranch() || analysisMetadataHolder.isPullRequest()) { + Component changedComponentTreeRoot = builder.buildChangedComponentTreeRoot(reportTreeRoot); + treeRootHolder.setRoot(changedComponentTreeRoot); + } else { + treeRootHolder.setRoot(reportTreeRoot); + } + analysisMetadataHolder.setBaseAnalysis(toAnalysis(baseAnalysis)); context.getStatistics().add("components", treeRootHolder.getSize()); diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/CoverageMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/CoverageMeasuresStep.java index e2a41566bb8..d7ac7ac7ea5 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/CoverageMeasuresStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/CoverageMeasuresStep.java @@ -68,7 +68,7 @@ public class CoverageMeasuresStep implements ComputationStep { public void execute(ComputationStep.Context context) { new PathAwareCrawler<>( FormulaExecutorComponentVisitor.newBuilder(metricRepository, measureRepository).buildFor(COVERAGE_FORMULAS)) - .visit(treeRootHolder.getRoot()); + .visit(treeRootHolder.getReportTreeRoot()); } private static class CodeCoverageFormula extends LinesAndConditionsWithUncoveredFormula { diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolderRule.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolderRule.java index 3c0a7015de2..5b11abf35d8 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolderRule.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolderRule.java @@ -25,4 +25,10 @@ public class MutableTreeRootHolderRule extends TreeRootHolderRule implements Mut delegate.setRoot(newRoot); return this; } + + @Override + public MutableTreeRootHolder setReportTreeRoot(Component newRoot) { + delegate.setReportTreeRoot(newRoot); + return this; + } } diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderRule.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderRule.java index 49b3adc1674..e0175a0b480 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderRule.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderRule.java @@ -33,6 +33,7 @@ public class TreeRootHolderRule extends ExternalResource implements TreeRootHold public TreeRootHolderRule setRoot(Component newRoot) { delegate = new TreeRootHolderImpl(); delegate.setRoot(newRoot); + delegate.setReportTreeRoot(newRoot); return this; } @@ -41,6 +42,11 @@ public class TreeRootHolderRule extends ExternalResource implements TreeRootHold return delegate.getRoot(); } + @Override + public Component getReportTreeRoot() { + return delegate.getReportTreeRoot(); + } + @Override public Component getComponentByRef(int ref) { return delegate.getComponentByRef(ref); diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStepTest.java index 6268cdd3af5..308b9914042 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStepTest.java @@ -38,6 +38,7 @@ import org.sonar.ce.task.projectanalysis.component.MutableTreeRootHolderRule; import org.sonar.ce.task.step.TestComputationStepContext; import org.sonar.db.DbClient; import org.sonar.db.DbTester; +import org.sonar.db.component.BranchType; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.SnapshotDto; import org.sonar.db.organization.OrganizationDto; @@ -77,6 +78,9 @@ public class BuildComponentTreeStepTest { private static final int FILE_2_REF = 5; private static final int DIR_REF_2 = 6; private static final int FILE_3_REF = 7; + private static final int LEAFLESS_MODULE_REF = 8; + private static final int LEAFLESS_DIR_REF = 9; + private static final int UNCHANGED_FILE_REF = 10; private static final String REPORT_PROJECT_KEY = "REPORT_PROJECT_KEY"; private static final String REPORT_MODULE_KEY = "MODULE_KEY"; @@ -84,6 +88,9 @@ public class BuildComponentTreeStepTest { private static final String REPORT_FILE_KEY_1 = "src/main/java/dir1/File1.java"; private static final String REPORT_DIR_KEY_2 = "src/main/java/dir2"; private static final String REPORT_FILE_KEY_2 = "src/main/java/dir2/File2.java"; + private static final String REPORT_LEAFLESS_MODULE_KEY = "LEAFLESS_MODULE_KEY"; + private static final String REPORT_LEAFLESS_DIR_KEY = "src/main/java/leafless"; + private static final String REPORT_UNCHANGED_FILE_KEY = "src/main/java/leafless/File3.java"; private static final long ANALYSIS_DATE = 123456789L; @@ -231,6 +238,94 @@ public class BuildComponentTreeStepTest { verifyComponent(FILE_1_REF, "generated", REPORT_MODULE_KEY + ":" + REPORT_FILE_KEY_1, null); } + @Test + @UseDataProvider("shortLivingBranchAndPullRequest") + public void prune_modules_and_directories_without_leaf_descendants_on_non_long_branch(BranchType branchType) { + Branch branch = mock(Branch.class); + when(branch.getName()).thenReturn("origin/feature"); + when(branch.isMain()).thenReturn(false); + when(branch.getType()).thenReturn(branchType); + when(branch.isLegacyFeature()).thenReturn(false); + when(branch.generateKey(any(), any())).thenReturn("generated"); + analysisMetadataHolder.setRootComponentRef(ROOT_REF) + .setAnalysisDate(ANALYSIS_DATE) + .setProject(Project.from(newPrivateProjectDto(newOrganizationDto()).setDbKey(REPORT_PROJECT_KEY))) + .setBranch(branch); + BuildComponentTreeStep underTest = new BuildComponentTreeStep(dbClient, reportReader, treeRootHolder, analysisMetadataHolder); + reportReader.putComponent(componentWithKey(ROOT_REF, PROJECT, REPORT_PROJECT_KEY, MODULE_REF, LEAFLESS_MODULE_REF)); + reportReader.putComponent(componentWithKey(MODULE_REF, MODULE, REPORT_MODULE_KEY, DIR_REF_1)); + reportReader.putComponent(componentWithPath(DIR_REF_1, DIRECTORY, REPORT_DIR_KEY_1, FILE_1_REF)); + reportReader.putComponent(componentWithPath(FILE_1_REF, FILE, REPORT_FILE_KEY_1)); + + reportReader.putComponent(componentWithKey(LEAFLESS_MODULE_REF, MODULE, REPORT_LEAFLESS_MODULE_KEY, LEAFLESS_DIR_REF)); + reportReader.putComponent(componentWithPath(LEAFLESS_DIR_REF, DIRECTORY, REPORT_LEAFLESS_DIR_KEY, UNCHANGED_FILE_REF)); + ScannerReport.Component unchangedFile = ScannerReport.Component.newBuilder() + .setType(FILE) + .setRef(UNCHANGED_FILE_REF) + .setPath(REPORT_UNCHANGED_FILE_KEY) + .setStatus(FileStatus.SAME) + .setLines(1) + .build(); + reportReader.putComponent(unchangedFile); + + underTest.execute(new TestComputationStepContext()); + + verifyComponent(ROOT_REF, "generated", REPORT_PROJECT_KEY, null); + verifyComponent(MODULE_REF, "generated", REPORT_MODULE_KEY, null); + verifyComponent(DIR_REF_1, "generated", REPORT_MODULE_KEY + ":" + REPORT_DIR_KEY_1, null); + verifyComponent(FILE_1_REF, "generated", REPORT_MODULE_KEY + ":" + REPORT_FILE_KEY_1, null); + + verifyComponentMissing(LEAFLESS_MODULE_REF); + verifyComponentMissing(LEAFLESS_DIR_REF); + verifyComponentMissing(UNCHANGED_FILE_REF); + } + + @Test + public void do_not_prune_modules_and_directories_without_leaf_descendants_on_long_branch() { + Branch branch = mock(Branch.class); + when(branch.getName()).thenReturn("origin/feature"); + when(branch.isMain()).thenReturn(false); + when(branch.getType()).thenReturn(BranchType.LONG); + when(branch.isLegacyFeature()).thenReturn(false); + when(branch.generateKey(any(), any())).thenReturn("generated"); + analysisMetadataHolder.setRootComponentRef(ROOT_REF) + .setAnalysisDate(ANALYSIS_DATE) + .setProject(Project.from(newPrivateProjectDto(newOrganizationDto()).setDbKey(REPORT_PROJECT_KEY))) + .setBranch(branch); + BuildComponentTreeStep underTest = new BuildComponentTreeStep(dbClient, reportReader, treeRootHolder, analysisMetadataHolder); + reportReader.putComponent(componentWithKey(ROOT_REF, PROJECT, REPORT_PROJECT_KEY, MODULE_REF, LEAFLESS_MODULE_REF)); + reportReader.putComponent(componentWithKey(MODULE_REF, MODULE, REPORT_MODULE_KEY, DIR_REF_1)); + reportReader.putComponent(componentWithPath(DIR_REF_1, DIRECTORY, REPORT_DIR_KEY_1, FILE_1_REF)); + reportReader.putComponent(componentWithPath(FILE_1_REF, FILE, REPORT_FILE_KEY_1)); + + reportReader.putComponent(componentWithKey(LEAFLESS_MODULE_REF, MODULE, REPORT_LEAFLESS_MODULE_KEY, LEAFLESS_DIR_REF)); + reportReader.putComponent(componentWithPath(LEAFLESS_DIR_REF, DIRECTORY, REPORT_LEAFLESS_DIR_KEY, UNCHANGED_FILE_REF)); + ScannerReport.Component unchangedFile = ScannerReport.Component.newBuilder() + .setType(FILE) + .setRef(UNCHANGED_FILE_REF) + .setPath(REPORT_UNCHANGED_FILE_KEY) + .setStatus(FileStatus.SAME) + .setLines(1) + .build(); + reportReader.putComponent(unchangedFile); + + underTest.execute(new TestComputationStepContext()); + + verifyComponent(ROOT_REF, "generated", REPORT_PROJECT_KEY, null); + verifyComponent(MODULE_REF, "generated", REPORT_MODULE_KEY, null); + verifyComponent(DIR_REF_1, "generated", REPORT_MODULE_KEY + ":" + REPORT_DIR_KEY_1, null); + verifyComponent(FILE_1_REF, "generated", REPORT_MODULE_KEY + ":" + REPORT_FILE_KEY_1, null); + + verifyComponent(LEAFLESS_MODULE_REF, "generated", REPORT_LEAFLESS_MODULE_KEY, null); + verifyComponent(LEAFLESS_DIR_REF, "generated", REPORT_LEAFLESS_MODULE_KEY + ":" + REPORT_LEAFLESS_DIR_KEY, null); + verifyComponent(UNCHANGED_FILE_REF, "generated", REPORT_LEAFLESS_MODULE_KEY + ":" + REPORT_UNCHANGED_FILE_KEY, null); + } + + @DataProvider + public static Object[][] shortLivingBranchAndPullRequest() { + return new Object[][] {{BranchType.SHORT}, {BranchType.PULL_REQUEST}}; + } + @Test public void generate_keys_when_using_existing_branch() { ComponentDto projectDto = dbTester.components().insertMainBranch(); @@ -416,6 +511,11 @@ public class BuildComponentTreeStepTest { } } + private void verifyComponentMissing(int ref) { + Map componentsByRef = indexAllComponentsInTreeByRef(treeRootHolder.getRoot()); + assertThat(componentsByRef.get(ref)).isNull(); + } + private static ScannerReport.Component component(int componentRef, ComponentType componentType, int... children) { return component(componentRef, componentType, null, null, children); }