]> source.dussan.org Git - sonarqube.git/commitdiff
SONAR-11125 Build a pruned component tree without unchanged files for selected visitors
authorJanos Gyerik <janos.gyerik@sonarsource.com>
Mon, 10 Sep 2018 15:40:51 +0000 (17:40 +0200)
committersonartech <sonartech@sonarsource.com>
Wed, 19 Sep 2018 08:51:42 +0000 (10:51 +0200)
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/Component.java
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolder.java
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolder.java
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderImpl.java
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/CoverageMeasuresStep.java
server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/MutableTreeRootHolderRule.java
server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/TreeRootHolderRule.java
server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStepTest.java

index af7a00e2ed51b09f18030ce1fd7d6e47554d684f..e76ee80f181da3ff9e325dd504d51c99a0384b45 100644 (file)
@@ -58,6 +58,10 @@ public interface Component {
     }
   }
 
+  /**
+   * On a long-living branch, CHANGED and ADDED are relative to the previous analysis.
+   * On a short-living branch and pull request, these are relative to the base branch.
+   */
   enum Status {
     UNAVAILABLE, SAME, CHANGED, ADDED
   }
index 427bb87bd520ab3574c2318618c9608cee963dd9..bc605c43db23ef0b6561340ab2f409767336a015 100644 (file)
@@ -20,6 +20,7 @@
 package org.sonar.ce.task.projectanalysis.component;
 
 import java.util.List;
+import java.util.Objects;
 import java.util.function.Function;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
@@ -145,6 +146,74 @@ public class ComponentTreeBuilder {
     }
   }
 
+  public Component buildChangedComponentTreeRoot(Component project) {
+    return buildChangedComponentTree(project);
+  }
+
+  private static ComponentImpl.Builder changedComponentBuilder(Component component) {
+    return ComponentImpl.builder(component.getType())
+      .setUuid(component.getUuid())
+      .setKey(component.getKey())
+      .setPublicKey(component.getPublicKey())
+      .setStatus(component.getStatus())
+      .setReportAttributes(component.getReportAttributes())
+      .setName(component.getName())
+      .setDescription(component.getDescription());
+  }
+
+  @Nullable
+  private static Component buildChangedComponentTree(Component component) {
+    switch (component.getType()) {
+      case PROJECT:
+        return buildChangedProject(component);
+
+      case MODULE:
+      case DIRECTORY:
+        return buildChangedIntermediate(component);
+
+      case FILE:
+        return buildChangedFile(component);
+
+      default:
+        throw new IllegalArgumentException(format("Unsupported component type '%s'", component.getType()));
+    }
+  }
+
+  private static Component buildChangedProject(Component component) {
+    return changedComponentBuilder(component)
+      .addChildren(buildChangedComponentChildren(component))
+      .build();
+  }
+
+  @Nullable
+  private static Component buildChangedIntermediate(Component component) {
+    List<Component> children = buildChangedComponentChildren(component);
+    if (children.isEmpty()) {
+      return null;
+    }
+    return changedComponentBuilder(component)
+      .addChildren(children)
+      .build();
+  }
+
+  @Nullable
+  private static Component buildChangedFile(Component component) {
+    if (component.getStatus() == Component.Status.SAME) {
+      return null;
+    }
+    return changedComponentBuilder(component)
+      .setFileAttributes(component.getFileAttributes())
+      .build();
+  }
+
+  private static List<Component> buildChangedComponentChildren(Component component) {
+    return component.getChildren()
+      .stream()
+      .map(ComponentTreeBuilder::buildChangedComponentTree)
+      .filter(Objects::nonNull)
+      .collect(MoreCollectors.toList());
+  }
+
   private void setNameAndDescription(ScannerReport.Component component, ComponentImpl.Builder builder) {
     if (branch.isMain()) {
       builder
index 880286b11617c92136d242f8eb0d7ded0408e525..e63bf33c176fc9e54fd7c6810729d495e82211b5 100644 (file)
@@ -22,13 +22,18 @@ package org.sonar.ce.task.projectanalysis.component;
 public interface MutableTreeRootHolder extends TreeRootHolder {
 
   /**
-   * Sets the root of the component tree in the TreeRootHolder. Settings a root more than once is allowed but it can
-   * never be set to {@code null}.
+   * Sets the root of the component tree in the TreeRootHolder.
    *
-   * @param newRoot a {@link Component}, can not be {@code null}
-   *                
    * @throws NullPointerException if {@code newRoot} is {@code null}
    * @throws IllegalStateException if root {@link Component} has already been set
    */
   MutableTreeRootHolder setRoot(Component newRoot);
+
+  /**
+   * Sets the root of the components that were in the scanner report in the TreeRootHolder.
+   *
+   * @throws NullPointerException if {@code newRoot} is {@code null}
+   * @throws IllegalStateException if extended tree root {@link Component} has already been set
+   */
+  MutableTreeRootHolder setReportTreeRoot(Component newRoot);
 }
index 85785ed633204fb2e8ef7dd651049652530949e9..347c7facad60a429c16d3b8e4335348ef7f5f01f 100644 (file)
@@ -29,10 +29,25 @@ public interface TreeRootHolder {
    * The root of the tree, for example the project or the portfolio.
    * With branches, it will refer to the root component of the branch.
    *
+   * On a short living branch or pull request, it contains ONLY:
+   * 1. The PROJECT component (tree root)
+   * 2. The FILE components whose status is not SAME
+   * 3. Intermediary MODULE and DIRECTORY components that lead to FILE leafs that are not SAME
+   *
    * @throws IllegalStateException if the holder is empty (ie. there is no root yet)
    */
   Component getRoot();
 
+  /**
+   * The root of the components that were in the scanner report.
+   * This tree may include components that are not persisted,
+   * just kept in memory for computation purposes, such as overall coverage
+   * in short living branches or pull requests.
+   *
+   * @throws IllegalStateException if the holder is empty (ie. there is no root yet)
+   */
+  Component getReportTreeRoot();
+
   /**
    * Return a component by its batch reference
    *
index 9598d93aabaf86553f4f660f8b9b0f8195091c88..91f47b639f1c6219e4d0de8909dd10a5f49f6c24 100644 (file)
@@ -36,6 +36,7 @@ public class TreeRootHolderImpl implements MutableTreeRootHolder {
   private Map<Integer, Component> componentsByRef;
 
   private Component root;
+  private Component extendedTreeRoot;
 
   @Override
   public MutableTreeRootHolder setRoot(Component root) {
@@ -50,6 +51,19 @@ public class TreeRootHolderImpl implements MutableTreeRootHolder {
     return this.root;
   }
 
+  @Override
+  public MutableTreeRootHolder setReportTreeRoot(Component root) {
+    checkState(this.extendedTreeRoot == null, "extended tree root can not be set twice in holder");
+    this.extendedTreeRoot = requireNonNull(root, "extended tree root can not be null");
+    return this;
+  }
+
+  @Override
+  public Component getReportTreeRoot() {
+    checkInitialized();
+    return this.extendedTreeRoot;
+  }
+
   @Override
   public Component getComponentByRef(int ref) {
     return getOptionalComponentByRef(ref)
index fee73596d1457edae4e29610951880194414b4c5..206f7d8a4c8fb62b9dc4433292d82d6ef0591836 100644 (file)
@@ -84,9 +84,17 @@ public class BuildComponentTreeStep implements ComputationStep {
         analysisMetadataHolder.getBranch(),
         baseAnalysis);
       String relativePathFromScmRoot = reportReader.readMetadata().getRelativePathFromScmRoot();
-      Component project = builder.buildProject(reportProject, relativePathFromScmRoot);
 
-      treeRootHolder.setRoot(project);
+      Component reportTreeRoot = builder.buildProject(reportProject, relativePathFromScmRoot);
+      treeRootHolder.setReportTreeRoot(reportTreeRoot);
+
+      if (analysisMetadataHolder.isShortLivingBranch() || analysisMetadataHolder.isPullRequest()) {
+        Component changedComponentTreeRoot = builder.buildChangedComponentTreeRoot(reportTreeRoot);
+        treeRootHolder.setRoot(changedComponentTreeRoot);
+      } else {
+        treeRootHolder.setRoot(reportTreeRoot);
+      }
+
       analysisMetadataHolder.setBaseAnalysis(toAnalysis(baseAnalysis));
 
       context.getStatistics().add("components", treeRootHolder.getSize());
index e2a41566bb8122e9b938de121b0ddfa3dade82d9..d7ac7ac7ea5af04a2c7d73ac50e8969d69062232 100644 (file)
@@ -68,7 +68,7 @@ public class CoverageMeasuresStep implements ComputationStep {
   public void execute(ComputationStep.Context context) {
     new PathAwareCrawler<>(
       FormulaExecutorComponentVisitor.newBuilder(metricRepository, measureRepository).buildFor(COVERAGE_FORMULAS))
-        .visit(treeRootHolder.getRoot());
+        .visit(treeRootHolder.getReportTreeRoot());
   }
 
   private static class CodeCoverageFormula extends LinesAndConditionsWithUncoveredFormula {
index 3c0a7015de2947873284795b164df701de30a342..5b11abf35d84569990773e59205478866c125263 100644 (file)
@@ -25,4 +25,10 @@ public class MutableTreeRootHolderRule extends TreeRootHolderRule implements Mut
     delegate.setRoot(newRoot);
     return this;
   }
+
+  @Override
+  public MutableTreeRootHolder setReportTreeRoot(Component newRoot) {
+    delegate.setReportTreeRoot(newRoot);
+    return this;
+  }
 }
index 49b3adc167471b7f55614f5d31ddfba838465a8a..e0175a0b4805de62f5bb8b1392d57d837989e07e 100644 (file)
@@ -33,6 +33,7 @@ public class TreeRootHolderRule extends ExternalResource implements TreeRootHold
   public TreeRootHolderRule setRoot(Component newRoot) {
     delegate = new TreeRootHolderImpl();
     delegate.setRoot(newRoot);
+    delegate.setReportTreeRoot(newRoot);
     return this;
   }
 
@@ -41,6 +42,11 @@ public class TreeRootHolderRule extends ExternalResource implements TreeRootHold
     return delegate.getRoot();
   }
 
+  @Override
+  public Component getReportTreeRoot() {
+    return delegate.getReportTreeRoot();
+  }
+
   @Override
   public Component getComponentByRef(int ref) {
     return delegate.getComponentByRef(ref);
index 6268cdd3af5d7414fc05fff543914fabe1f8287e..308b99140429ab29a76e406cf317820dc28ce82f 100644 (file)
@@ -38,6 +38,7 @@ import org.sonar.ce.task.projectanalysis.component.MutableTreeRootHolderRule;
 import org.sonar.ce.task.step.TestComputationStepContext;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbTester;
+import org.sonar.db.component.BranchType;
 import org.sonar.db.component.ComponentDto;
 import org.sonar.db.component.SnapshotDto;
 import org.sonar.db.organization.OrganizationDto;
@@ -77,6 +78,9 @@ public class BuildComponentTreeStepTest {
   private static final int FILE_2_REF = 5;
   private static final int DIR_REF_2 = 6;
   private static final int FILE_3_REF = 7;
+  private static final int LEAFLESS_MODULE_REF = 8;
+  private static final int LEAFLESS_DIR_REF = 9;
+  private static final int UNCHANGED_FILE_REF = 10;
 
   private static final String REPORT_PROJECT_KEY = "REPORT_PROJECT_KEY";
   private static final String REPORT_MODULE_KEY = "MODULE_KEY";
@@ -84,6 +88,9 @@ public class BuildComponentTreeStepTest {
   private static final String REPORT_FILE_KEY_1 = "src/main/java/dir1/File1.java";
   private static final String REPORT_DIR_KEY_2 = "src/main/java/dir2";
   private static final String REPORT_FILE_KEY_2 = "src/main/java/dir2/File2.java";
+  private static final String REPORT_LEAFLESS_MODULE_KEY = "LEAFLESS_MODULE_KEY";
+  private static final String REPORT_LEAFLESS_DIR_KEY = "src/main/java/leafless";
+  private static final String REPORT_UNCHANGED_FILE_KEY = "src/main/java/leafless/File3.java";
 
   private static final long ANALYSIS_DATE = 123456789L;
 
@@ -231,6 +238,94 @@ public class BuildComponentTreeStepTest {
     verifyComponent(FILE_1_REF, "generated", REPORT_MODULE_KEY + ":" + REPORT_FILE_KEY_1, null);
   }
 
+  @Test
+  @UseDataProvider("shortLivingBranchAndPullRequest")
+  public void prune_modules_and_directories_without_leaf_descendants_on_non_long_branch(BranchType branchType) {
+    Branch branch = mock(Branch.class);
+    when(branch.getName()).thenReturn("origin/feature");
+    when(branch.isMain()).thenReturn(false);
+    when(branch.getType()).thenReturn(branchType);
+    when(branch.isLegacyFeature()).thenReturn(false);
+    when(branch.generateKey(any(), any())).thenReturn("generated");
+    analysisMetadataHolder.setRootComponentRef(ROOT_REF)
+      .setAnalysisDate(ANALYSIS_DATE)
+      .setProject(Project.from(newPrivateProjectDto(newOrganizationDto()).setDbKey(REPORT_PROJECT_KEY)))
+      .setBranch(branch);
+    BuildComponentTreeStep underTest = new BuildComponentTreeStep(dbClient, reportReader, treeRootHolder, analysisMetadataHolder);
+    reportReader.putComponent(componentWithKey(ROOT_REF, PROJECT, REPORT_PROJECT_KEY, MODULE_REF, LEAFLESS_MODULE_REF));
+    reportReader.putComponent(componentWithKey(MODULE_REF, MODULE, REPORT_MODULE_KEY, DIR_REF_1));
+    reportReader.putComponent(componentWithPath(DIR_REF_1, DIRECTORY, REPORT_DIR_KEY_1, FILE_1_REF));
+    reportReader.putComponent(componentWithPath(FILE_1_REF, FILE, REPORT_FILE_KEY_1));
+
+    reportReader.putComponent(componentWithKey(LEAFLESS_MODULE_REF, MODULE, REPORT_LEAFLESS_MODULE_KEY, LEAFLESS_DIR_REF));
+    reportReader.putComponent(componentWithPath(LEAFLESS_DIR_REF, DIRECTORY, REPORT_LEAFLESS_DIR_KEY, UNCHANGED_FILE_REF));
+    ScannerReport.Component unchangedFile = ScannerReport.Component.newBuilder()
+      .setType(FILE)
+      .setRef(UNCHANGED_FILE_REF)
+      .setPath(REPORT_UNCHANGED_FILE_KEY)
+      .setStatus(FileStatus.SAME)
+      .setLines(1)
+      .build();
+    reportReader.putComponent(unchangedFile);
+
+    underTest.execute(new TestComputationStepContext());
+
+    verifyComponent(ROOT_REF, "generated", REPORT_PROJECT_KEY, null);
+    verifyComponent(MODULE_REF, "generated", REPORT_MODULE_KEY, null);
+    verifyComponent(DIR_REF_1, "generated", REPORT_MODULE_KEY + ":" + REPORT_DIR_KEY_1, null);
+    verifyComponent(FILE_1_REF, "generated", REPORT_MODULE_KEY + ":" + REPORT_FILE_KEY_1, null);
+
+    verifyComponentMissing(LEAFLESS_MODULE_REF);
+    verifyComponentMissing(LEAFLESS_DIR_REF);
+    verifyComponentMissing(UNCHANGED_FILE_REF);
+  }
+
+  @Test
+  public void do_not_prune_modules_and_directories_without_leaf_descendants_on_long_branch() {
+    Branch branch = mock(Branch.class);
+    when(branch.getName()).thenReturn("origin/feature");
+    when(branch.isMain()).thenReturn(false);
+    when(branch.getType()).thenReturn(BranchType.LONG);
+    when(branch.isLegacyFeature()).thenReturn(false);
+    when(branch.generateKey(any(), any())).thenReturn("generated");
+    analysisMetadataHolder.setRootComponentRef(ROOT_REF)
+      .setAnalysisDate(ANALYSIS_DATE)
+      .setProject(Project.from(newPrivateProjectDto(newOrganizationDto()).setDbKey(REPORT_PROJECT_KEY)))
+      .setBranch(branch);
+    BuildComponentTreeStep underTest = new BuildComponentTreeStep(dbClient, reportReader, treeRootHolder, analysisMetadataHolder);
+    reportReader.putComponent(componentWithKey(ROOT_REF, PROJECT, REPORT_PROJECT_KEY, MODULE_REF, LEAFLESS_MODULE_REF));
+    reportReader.putComponent(componentWithKey(MODULE_REF, MODULE, REPORT_MODULE_KEY, DIR_REF_1));
+    reportReader.putComponent(componentWithPath(DIR_REF_1, DIRECTORY, REPORT_DIR_KEY_1, FILE_1_REF));
+    reportReader.putComponent(componentWithPath(FILE_1_REF, FILE, REPORT_FILE_KEY_1));
+
+    reportReader.putComponent(componentWithKey(LEAFLESS_MODULE_REF, MODULE, REPORT_LEAFLESS_MODULE_KEY, LEAFLESS_DIR_REF));
+    reportReader.putComponent(componentWithPath(LEAFLESS_DIR_REF, DIRECTORY, REPORT_LEAFLESS_DIR_KEY, UNCHANGED_FILE_REF));
+    ScannerReport.Component unchangedFile = ScannerReport.Component.newBuilder()
+      .setType(FILE)
+      .setRef(UNCHANGED_FILE_REF)
+      .setPath(REPORT_UNCHANGED_FILE_KEY)
+      .setStatus(FileStatus.SAME)
+      .setLines(1)
+      .build();
+    reportReader.putComponent(unchangedFile);
+
+    underTest.execute(new TestComputationStepContext());
+
+    verifyComponent(ROOT_REF, "generated", REPORT_PROJECT_KEY, null);
+    verifyComponent(MODULE_REF, "generated", REPORT_MODULE_KEY, null);
+    verifyComponent(DIR_REF_1, "generated", REPORT_MODULE_KEY + ":" + REPORT_DIR_KEY_1, null);
+    verifyComponent(FILE_1_REF, "generated", REPORT_MODULE_KEY + ":" + REPORT_FILE_KEY_1, null);
+
+    verifyComponent(LEAFLESS_MODULE_REF, "generated", REPORT_LEAFLESS_MODULE_KEY, null);
+    verifyComponent(LEAFLESS_DIR_REF, "generated", REPORT_LEAFLESS_MODULE_KEY + ":" + REPORT_LEAFLESS_DIR_KEY, null);
+    verifyComponent(UNCHANGED_FILE_REF, "generated", REPORT_LEAFLESS_MODULE_KEY + ":" + REPORT_UNCHANGED_FILE_KEY, null);
+  }
+
+  @DataProvider
+  public static Object[][] shortLivingBranchAndPullRequest() {
+    return new Object[][] {{BranchType.SHORT}, {BranchType.PULL_REQUEST}};
+  }
+
   @Test
   public void generate_keys_when_using_existing_branch() {
     ComponentDto projectDto = dbTester.components().insertMainBranch();
@@ -416,6 +511,11 @@ public class BuildComponentTreeStepTest {
     }
   }
 
+  private void verifyComponentMissing(int ref) {
+    Map<Integer, Component> componentsByRef = indexAllComponentsInTreeByRef(treeRootHolder.getRoot());
+    assertThat(componentsByRef.get(ref)).isNull();
+  }
+
   private static ScannerReport.Component component(int componentRef, ComponentType componentType, int... children) {
     return component(componentRef, componentType, null, null, children);
   }