SONAR-9352 do not let quality gate fail for very small changesets

server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java

@@ -116,6 +116,7 @@ import org.sonar.server.computation.task.projectanalysis.source.LastCommitVisito
 import org.sonar.server.computation.task.projectanalysis.source.SourceHashRepositoryImpl;
 import org.sonar.server.computation.task.projectanalysis.source.SourceLinesRepositoryImpl;
 import org.sonar.server.computation.task.projectanalysis.step.ReportComputationSteps;
+import org.sonar.server.computation.task.projectanalysis.step.SmallChangesetQualityGateSpecialCase;
 import org.sonar.server.computation.task.projectanalysis.webhook.WebhookModule;
 import org.sonar.server.computation.task.step.ComputationStepExecutor;
 import org.sonar.server.computation.task.step.ComputationSteps;
@@ -267,6 +268,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop
 
       BranchLoader.class,
       MeasureToMeasureDto.class,
+      SmallChangesetQualityGateSpecialCase.class,
 
       // webhooks
       WebhookModule.class);

server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/QualityGateMeasuresStep.java

@@ -83,17 +83,19 @@ public class QualityGateMeasuresStep implements ComputationStep {
   private final MeasureRepository measureRepository;
   private final MetricRepository metricRepository;
   private final EvaluationResultTextConverter evaluationResultTextConverter;
+  private final SmallChangesetQualityGateSpecialCase smallChangesetQualityGateSpecialCase;
 
   public QualityGateMeasuresStep(TreeRootHolder treeRootHolder,
     QualityGateHolder qualityGateHolder, MutableQualityGateStatusHolder qualityGateStatusHolder,
     MeasureRepository measureRepository, MetricRepository metricRepository,
-    EvaluationResultTextConverter evaluationResultTextConverter) {
+    EvaluationResultTextConverter evaluationResultTextConverter, SmallChangesetQualityGateSpecialCase smallChangesetQualityGateSpecialCase) {
     this.treeRootHolder = treeRootHolder;
     this.qualityGateHolder = qualityGateHolder;
     this.qualityGateStatusHolder = qualityGateStatusHolder;
     this.evaluationResultTextConverter = evaluationResultTextConverter;
     this.measureRepository = measureRepository;
     this.metricRepository = metricRepository;
+    this.smallChangesetQualityGateSpecialCase = smallChangesetQualityGateSpecialCase;
   }
 
   @Override
@@ -179,16 +181,17 @@ public class QualityGateMeasuresStep implements ComputationStep {
         continue;
       }
 
-      MetricEvaluationResult metricEvaluationResult = evaluateQualityGate(measure.get(), entry.getValue());
-      String text = evaluationResultTextConverter.asText(metricEvaluationResult.condition, metricEvaluationResult.evaluationResult);
+      final MetricEvaluationResult metricEvaluationResult = evaluateQualityGate(measure.get(), entry.getValue());
+      final MetricEvaluationResult finalMetricEvaluationResult = smallChangesetQualityGateSpecialCase.applyIfNeeded(project, metricEvaluationResult);
+      String text = evaluationResultTextConverter.asText(finalMetricEvaluationResult.condition, finalMetricEvaluationResult.evaluationResult);
       builder.addLabel(text);
 
       Measure updatedMeasure = Measure.updatedMeasureBuilder(measure.get())
-        .setQualityGateStatus(new QualityGateStatus(metricEvaluationResult.evaluationResult.getLevel(), text))
+        .setQualityGateStatus(new QualityGateStatus(finalMetricEvaluationResult.evaluationResult.getLevel(), text))
         .create();
       measureRepository.update(project, metric, updatedMeasure);
 
-      builder.addEvaluatedCondition(metricEvaluationResult);
+      builder.addEvaluatedCondition(finalMetricEvaluationResult);
     }
   }
 
@@ -268,11 +271,11 @@ public class QualityGateMeasuresStep implements ComputationStep {
     }
   }
 
-  private static class MetricEvaluationResult {
-    private final EvaluationResult evaluationResult;
-    private final Condition condition;
+  static class MetricEvaluationResult {
+    final EvaluationResult evaluationResult;
+    final Condition condition;
 
-    private MetricEvaluationResult(EvaluationResult evaluationResult, Condition condition) {
+    MetricEvaluationResult(EvaluationResult evaluationResult, Condition condition) {
       this.evaluationResult = evaluationResult;
       this.condition = condition;
     }

server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/SmallChangesetQualityGateSpecialCase.java

@@ -0,0 +1,82 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.computation.task.projectanalysis.step;
+
+import java.util.Collection;
+import javax.annotation.Nullable;
+import org.sonar.api.measures.CoreMetrics;
+import org.sonar.server.computation.task.projectanalysis.component.Component;
+import org.sonar.server.computation.task.projectanalysis.measure.Measure;
+import org.sonar.server.computation.task.projectanalysis.measure.MeasureRepository;
+import org.sonar.server.computation.task.projectanalysis.metric.MetricRepository;
+import org.sonar.server.computation.task.projectanalysis.qualitygate.EvaluationResult;
+
+import static java.util.Arrays.asList;
+
+public class SmallChangesetQualityGateSpecialCase {
+
+  /**
+   * Some metrics will only produce warnings (never errors) on very small change sets.
+   */
+  private static final Collection<String> METRICS_THAT_CAN_ONLY_PRODUCE_WARNINGS_ON_SMALL_CHANGESETS = asList(
+    CoreMetrics.NEW_COVERAGE_KEY,
+    CoreMetrics.NEW_LINE_COVERAGE_KEY,
+    CoreMetrics.NEW_BRANCH_COVERAGE_KEY,
+    CoreMetrics.NEW_DUPLICATED_LINES_DENSITY_KEY,
+    CoreMetrics.NEW_DUPLICATED_LINES_KEY,
+    CoreMetrics.NEW_BLOCKS_DUPLICATED_KEY
+  );
+  private static final int MAXIMUM_NEW_LINES_FOR_SMALL_CHANGESETS = 19;
+
+  private final MeasureRepository measureRepository;
+  private final MetricRepository metricRepository;
+
+  public SmallChangesetQualityGateSpecialCase(MeasureRepository measureRepository, MetricRepository metricRepository) {
+    this.measureRepository = measureRepository;
+    this.metricRepository = metricRepository;
+  }
+
+  QualityGateMeasuresStep.MetricEvaluationResult applyIfNeeded(Component project, @Nullable QualityGateMeasuresStep.MetricEvaluationResult metricEvaluationResult) {
+    if (metricEvaluationResult == null) {
+      return metricEvaluationResult;
+    }
+    if (metricEvaluationResult.evaluationResult.getLevel() == Measure.Level.OK) {
+      return metricEvaluationResult;
+    }
+    if (!METRICS_THAT_CAN_ONLY_PRODUCE_WARNINGS_ON_SMALL_CHANGESETS.contains(metricEvaluationResult.condition.getMetric().getKey())) {
+      return metricEvaluationResult;
+    }
+    if (!isSmallChangeset(project)) {
+      return metricEvaluationResult;
+    }
+    return calculateModifiedResult(metricEvaluationResult);
+  }
+
+  private QualityGateMeasuresStep.MetricEvaluationResult calculateModifiedResult(@Nullable QualityGateMeasuresStep.MetricEvaluationResult metricEvaluationResult) {
+    return new QualityGateMeasuresStep.MetricEvaluationResult(
+      new EvaluationResult(Measure.Level.OK, metricEvaluationResult.evaluationResult.getValue()), metricEvaluationResult.condition);
+  }
+
+  private boolean isSmallChangeset(Component project) {
+    return measureRepository.getRawMeasure(project, metricRepository.getByKey(CoreMetrics.NEW_LINES_KEY))
+      .transform(newLines -> newLines.hasVariation() && newLines.getVariation() <= MAXIMUM_NEW_LINES_FOR_SMALL_CHANGESETS)
+      .or(false);
+  }
+}

server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/QualityGateMeasuresStepTest.java

@@ -89,7 +89,7 @@ public class QualityGateMeasuresStepTest {
 
   private EvaluationResultTextConverter resultTextConverter = mock(EvaluationResultTextConverter.class);
   private QualityGateMeasuresStep underTest = new QualityGateMeasuresStep(treeRootHolder, qualityGateHolder, qualityGateStatusHolder, measureRepository, metricRepository,
-    resultTextConverter);
+    resultTextConverter, new SmallChangesetQualityGateSpecialCase(measureRepository, metricRepository));
 
   @Before
   public void setUp() {

server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/SmallChangesetQualityGateSpecialCaseTest.java

@@ -0,0 +1,142 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.computation.task.projectanalysis.step;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.measures.CoreMetrics;
+import org.sonar.server.computation.task.projectanalysis.component.Component;
+import org.sonar.server.computation.task.projectanalysis.component.TreeRootHolderRule;
+import org.sonar.server.computation.task.projectanalysis.measure.Measure;
+import org.sonar.server.computation.task.projectanalysis.measure.MeasureRepositoryRule;
+import org.sonar.server.computation.task.projectanalysis.metric.Metric;
+import org.sonar.server.computation.task.projectanalysis.metric.MetricRepositoryRule;
+import org.sonar.server.computation.task.projectanalysis.qualitygate.Condition;
+import org.sonar.server.computation.task.projectanalysis.qualitygate.EvaluationResult;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.sonar.api.measures.CoreMetrics.NEW_BUGS_KEY;
+import static org.sonar.api.measures.CoreMetrics.NEW_COVERAGE_KEY;
+import static org.sonar.server.computation.task.projectanalysis.component.ReportComponent.builder;
+import static org.sonar.server.computation.task.projectanalysis.measure.Measure.Level.ERROR;
+import static org.sonar.server.computation.task.projectanalysis.measure.Measure.Level.OK;
+import static org.sonar.server.computation.task.projectanalysis.measure.Measure.Level.WARN;
+import static org.sonar.server.computation.task.projectanalysis.measure.Measure.newMeasureBuilder;
+
+public class SmallChangesetQualityGateSpecialCaseTest {
+
+  public static final int PROJECT_REF = 1234;
+  @Rule
+  public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule();
+  @Rule
+  public MetricRepositoryRule metricRepository = new MetricRepositoryRule()
+    .add(CoreMetrics.NEW_LINES)
+    .add(CoreMetrics.NEW_COVERAGE)
+    .add(CoreMetrics.NEW_BUGS);
+  @Rule
+  public MeasureRepositoryRule measureRepository = MeasureRepositoryRule.create(treeRootHolder, metricRepository);
+  private final SmallChangesetQualityGateSpecialCase underTest = new SmallChangesetQualityGateSpecialCase(measureRepository, metricRepository);
+
+  @Test
+  public void ignore_errors_about_new_coverage_for_small_changesets() throws Exception {
+    QualityGateMeasuresStep.MetricEvaluationResult metricEvaluationResult = generateEvaluationResult(NEW_COVERAGE_KEY, ERROR);
+    Component project = generateNewRootProject();
+    measureRepository.addRawMeasure(PROJECT_REF, CoreMetrics.NEW_LINES_KEY, newMeasureBuilder().setVariation(19).create(1000));
+
+    QualityGateMeasuresStep.MetricEvaluationResult result = underTest.applyIfNeeded(project, metricEvaluationResult);
+
+    assertThat(result.evaluationResult.getLevel()).isSameAs(OK);
+  }
+
+  @Test
+  public void ignore_warnings_about_new_coverage_for_small_changesets() throws Exception {
+    QualityGateMeasuresStep.MetricEvaluationResult metricEvaluationResult = generateEvaluationResult(NEW_COVERAGE_KEY, WARN);
+    Component project = generateNewRootProject();
+    measureRepository.addRawMeasure(PROJECT_REF, CoreMetrics.NEW_LINES_KEY, newMeasureBuilder().setVariation(19).create(1000));
+
+    QualityGateMeasuresStep.MetricEvaluationResult result = underTest.applyIfNeeded(project, metricEvaluationResult);
+
+    assertThat(result.evaluationResult.getLevel()).isSameAs(OK);
+  }
+
+  @Test
+  public void should_not_change_for_bigger_changesets() throws Exception {
+    QualityGateMeasuresStep.MetricEvaluationResult metricEvaluationResult = generateEvaluationResult(NEW_COVERAGE_KEY, ERROR);
+    Component project = generateNewRootProject();
+    measureRepository.addRawMeasure(PROJECT_REF, CoreMetrics.NEW_LINES_KEY, newMeasureBuilder().setVariation(20).create(1000));
+
+    QualityGateMeasuresStep.MetricEvaluationResult result = underTest.applyIfNeeded(project, metricEvaluationResult);
+
+    assertThat(result.evaluationResult.getLevel()).isSameAs(ERROR);
+  }
+
+  @Test
+  public void should_not_change_issue_related_metrics() throws Exception {
+    QualityGateMeasuresStep.MetricEvaluationResult metricEvaluationResult = generateEvaluationResult(NEW_BUGS_KEY, ERROR);
+    Component project = generateNewRootProject();
+    measureRepository.addRawMeasure(PROJECT_REF, CoreMetrics.NEW_LINES_KEY, newMeasureBuilder().setVariation(19).create(1000));
+
+    QualityGateMeasuresStep.MetricEvaluationResult result = underTest.applyIfNeeded(project, metricEvaluationResult);
+
+    assertThat(result.evaluationResult.getLevel()).isSameAs(ERROR);
+  }
+
+  @Test
+  public void should_not_change_green_conditions() throws Exception {
+    QualityGateMeasuresStep.MetricEvaluationResult metricEvaluationResult = generateEvaluationResult(NEW_BUGS_KEY, OK);
+    Component project = generateNewRootProject();
+    measureRepository.addRawMeasure(PROJECT_REF, CoreMetrics.NEW_LINES_KEY, newMeasureBuilder().setVariation(19).create(1000));
+
+    QualityGateMeasuresStep.MetricEvaluationResult result = underTest.applyIfNeeded(project, metricEvaluationResult);
+
+    assertThat(result.evaluationResult.getLevel()).isSameAs(OK);
+  }
+
+  @Test
+  public void should_not_change_quality_gate_if_new_lines_is_not_defined() throws Exception {
+    QualityGateMeasuresStep.MetricEvaluationResult metricEvaluationResult = generateEvaluationResult(NEW_COVERAGE_KEY, ERROR);
+    Component project = generateNewRootProject();
+
+    QualityGateMeasuresStep.MetricEvaluationResult result = underTest.applyIfNeeded(project, metricEvaluationResult);
+
+    assertThat(result.evaluationResult.getLevel()).isSameAs(ERROR);
+  }
+
+  @Test
+  public void should_silently_ignore_null_values() throws Exception {
+
+    QualityGateMeasuresStep.MetricEvaluationResult result = underTest.applyIfNeeded(mock(Component.class), null);
+
+    assertThat(result).isNull();
+  }
+
+  private Component generateNewRootProject() {
+    treeRootHolder.setRoot(builder(Component.Type.PROJECT, PROJECT_REF).build());
+    return treeRootHolder.getRoot();
+  }
+
+  private QualityGateMeasuresStep.MetricEvaluationResult generateEvaluationResult(String metric, Measure.Level level) {
+    Metric newCoverageMetric = metricRepository.getByKey(metric);
+    Condition condition = new Condition(newCoverageMetric, "LT", "80", "90", true);
+    EvaluationResult evaluationResult = new EvaluationResult(level, mock(Comparable.class));
+    return new QualityGateMeasuresStep.MetricEvaluationResult(evaluationResult, condition);
+  }
+}

tests/projects/qualitygate/small-changesets/v1-1000-lines/sonar-project.properties

@@ -0,0 +1,4 @@
+sonar.projectKey=sample
+sonar.projectName=Sample
+sonar.projectVersion=1.0-SNAPSHOT
+sonar.sources=src

tests/projects/qualitygate/small-changesets/v1-1000-lines/src/sample/Sample.xoo.measures

@@ -0,0 +1,3 @@
+ncloc:1000
+complexity:7
+test_execution_time:630

tests/projects/qualitygate/small-changesets/v2-1019-lines/sonar-project.properties

@@ -0,0 +1,4 @@
+sonar.projectKey=sample
+sonar.projectName=Sample
+sonar.projectVersion=2.0-SNAPSHOT
+sonar.sources=src

tests/projects/qualitygate/small-changesets/v2-1019-lines/src/sample/Sample.xoo.measures

@@ -0,0 +1,3 @@
+ncloc:1019
+complexity:7
+test_execution_time:630

tests/projects/qualitygate/small-changesets/v2-1020-lines/sonar-project.properties

@@ -0,0 +1,4 @@
+sonar.projectKey=sample
+sonar.projectName=Sample
+sonar.projectVersion=2.0-SNAPSHOT
+sonar.sources=src

tests/projects/qualitygate/small-changesets/v2-1020-lines/src/sample/Sample.xoo.measures

@@ -0,0 +1,3 @@
+ncloc:1019
+complexity:7
+test_execution_time:630

tests/src/test/java/org/sonarqube/tests/qualityGate/QualityGateForSmallChangesetsTest.java

@@ -0,0 +1,106 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarqube.tests.qualityGate;
+
+import com.sonar.orchestrator.Orchestrator;
+import com.sonar.orchestrator.build.SonarScanner;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonarqube.tests.Category6Suite;
+import org.sonarqube.tests.Tester;
+import org.sonarqube.ws.Organizations;
+import org.sonarqube.ws.WsProjects.CreateWsResponse.Project;
+import org.sonarqube.ws.WsQualityGates;
+import org.sonarqube.ws.WsQualityGates.CreateWsResponse;
+import org.sonarqube.ws.WsUsers;
+import org.sonarqube.ws.client.qualitygate.CreateConditionRequest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static util.ItUtils.getMeasure;
+import static util.ItUtils.projectDir;
+
+public class QualityGateForSmallChangesetsTest {
+
+  @ClassRule
+  public static Orchestrator orchestrator = Category6Suite.ORCHESTRATOR;
+
+  @Rule
+  public Tester tester = new Tester(orchestrator);
+
+  @Test
+  public void do_not_fail_quality_gate_with_poor_LEAK_coverage_and_a_max_of_19_lines_of_NEW_code() throws Exception {
+    Organizations.Organization organization = tester.organizations().generate();
+    Project project = tester.projects().generate(organization);
+    CreateWsResponse qualityGate = tester.qGates().generate();
+    tester.qGates().associateProject(qualityGate, project);
+    WsQualityGates.CreateConditionWsResponse condition = tester.wsClient().qualityGates().createCondition(CreateConditionRequest.builder()
+      .setQualityGateId(qualityGate.getId())
+      .setMetricKey("new_coverage")
+      .setOperator("LT")
+      .setWarning("90")
+      .setError("80")
+      .setPeriod(1)
+      .build());
+
+    tester.settings().setProjectSetting(project.getKey(), "sonar.leak.period", "previous_version");
+    String password = "password1";
+    WsUsers.CreateWsResponse.User user = tester.users().generateAdministrator(organization, u -> u.setPassword(password));
+
+    SonarScanner analysis = SonarScanner
+      .create(projectDir("qualitygate/small-changesets/v1-1000-lines"))
+      .setProperty("sonar.projectKey", project.getKey())
+      .setProperty("sonar.organization", organization.getKey())
+      .setProperty("sonar.login", user.getLogin())
+      .setProperty("sonar.password", password)
+      .setProperty("sonar.scm.provider", "xoo")
+      .setProperty("sonar.scm.disabled", "false")
+      .setProperty("sonar.projectDate", "2013-04-01")
+      .setDebugLogs(true);
+    orchestrator.executeBuild(analysis);
+    assertThat(getMeasure(orchestrator, project.getKey(), "alert_status").getValue()).isEqualTo("OK");
+
+    SonarScanner analysis2 = SonarScanner
+      .create(projectDir("qualitygate/small-changesets/v2-1019-lines"))
+      .setProperty("sonar.projectKey", project.getKey())
+      .setProperty("sonar.organization", organization.getKey())
+      .setProperty("sonar.login", user.getLogin())
+      .setProperty("sonar.password", password)
+      .setProperty("sonar.scm.provider", "xoo")
+      .setProperty("sonar.scm.disabled", "false")
+      .setProperty("sonar.projectDate", "2014-04-01")
+      .setDebugLogs(true);
+    orchestrator.executeBuild(analysis2);
+    assertThat(getMeasure(orchestrator, project.getKey(), "alert_status").getValue()).isEqualTo("OK");
+
+    SonarScanner analysis4 = SonarScanner
+      .create(projectDir("qualitygate/small-changesets/v2-1020-lines"))
+      .setProperty("sonar.projectKey", project.getKey())
+      .setProperty("sonar.organization", organization.getKey())
+      .setProperty("sonar.login", user.getLogin())
+      .setProperty("sonar.password", password)
+      .setProperty("sonar.scm.provider", "xoo")
+      .setProperty("sonar.scm.disabled", "false")
+      .setProperty("sonar.projectDate", "2014-04-02")
+      .setDebugLogs(true);
+    orchestrator.executeBuild(analysis4);
+    assertThat(getMeasure(orchestrator, project.getKey(), "alert_status").getValue()).isEqualTo("ERROR");
+  }
+}