SONAR-12140 Compute number of projects in warning in a daemon

server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java

@@ -349,7 +349,9 @@ public class ProjectMeasuresIndex {
   private Map<String, QueryBuilder> createFilters(ProjectMeasuresQuery query) {
     Map<String, QueryBuilder> filters = new HashMap<>();
     filters.put("__indexType", termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName()));
-    filters.put("__authorization", authorizationTypeSupport.createQueryFilter());
+    if (!query.isIgnoreAuthorization()) {
+      filters.put("__authorization", authorizationTypeSupport.createQueryFilter());
+    }
     Multimap<String, MetricCriterion> metricCriterionMultimap = ArrayListMultimap.create();
     query.getMetricCriteria().forEach(metricCriterion -> metricCriterionMultimap.put(metricCriterion.getMetricKey(), metricCriterion));
     metricCriterionMultimap.asMap().forEach((key, value) -> {

server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresQuery.java

@@ -45,6 +45,7 @@ public class ProjectMeasuresQuery {
   private String sort = SORT_BY_NAME;
   private boolean asc = true;
   private String queryText;
+  private boolean ignoreAuthorization;
 
   public ProjectMeasuresQuery addMetricCriterion(MetricCriterion metricCriterion) {
     this.metricCriteria.add(metricCriterion);
@@ -127,6 +128,15 @@ public class ProjectMeasuresQuery {
     return this;
   }
 
+  public boolean isIgnoreAuthorization() {
+    return ignoreAuthorization;
+  }
+
+  public ProjectMeasuresQuery setIgnoreAuthorization(boolean ignoreAuthorization) {
+    this.ignoreAuthorization = ignoreAuthorization;
+    return this;
+  }
+
   public static class MetricCriterion {
     private final String metricKey;
     private final Operator operator;

server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java

@@ -27,6 +27,7 @@ import org.sonar.server.es.IndexerStartupTask;
 import org.sonar.server.organization.DefaultOrganizationEnforcer;
 import org.sonar.server.platform.ServerLifecycleNotifier;
 import org.sonar.server.platform.web.RegisterServletFilters;
+import org.sonar.server.qualitygate.ProjectsInWarningDaemon;
 import org.sonar.server.qualitygate.RegisterQualityGates;
 import org.sonar.server.qualityprofile.BuiltInQProfileInsertImpl;
 import org.sonar.server.qualityprofile.BuiltInQProfileLoader;
@@ -81,6 +82,8 @@ public class PlatformLevelStartup extends PlatformLevel {
       protected void doPrivileged() {
         PlatformLevelStartup.super.start();
         getOptional(IndexerStartupTask.class).ifPresent(IndexerStartupTask::execute);
+        // Need to be executed after indexing as it executes an ES query
+        get(ProjectsInWarningDaemon.class).notifyStart();
         get(ServerLifecycleNotifier.class).notifyStart();
         get(ProcessCommandWrapper.class).notifyOperational();
         get(WebServerRuleFinder.class).stopCaching();

server/sonar-server/src/main/java/org/sonar/server/qualitygate/ProjectsInWarning.java

@@ -0,0 +1,44 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.qualitygate;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * Store number of projects in warning in order for the web service api/components/search to know if warning value should be return in the quality gate facet.
+ * The value is updated each time the daemon {@link ProjectsInWarningDaemon} is executed
+ */
+public class ProjectsInWarning {
+
+  private Long projectsInWarning;
+
+  public void update(long projectsInWarning) {
+    this.projectsInWarning = projectsInWarning;
+  }
+
+  public long count() {
+    checkArgument(isInitialized(), "Initialization has not be done");
+    return projectsInWarning;
+  }
+
+  boolean isInitialized() {
+    return projectsInWarning != null;
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/qualitygate/ProjectsInWarningDaemon.java

@@ -0,0 +1,149 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.qualitygate;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import org.picocontainer.Startable;
+import org.sonar.api.config.Configuration;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.server.es.SearchOptions;
+import org.sonar.server.measure.index.ProjectMeasuresIndex;
+import org.sonar.server.measure.index.ProjectMeasuresQuery;
+import org.sonar.server.util.GlobalLockManager;
+
+import static org.sonar.api.measures.Metric.Level.WARN;
+
+/**
+ * This class is regularly checking the number of projects in warning state, in order to not return the "Warning" value in the quality gate facet of the Projects page when there are no more projects in warning.
+ * 
+ * @see <a href="https://jira.sonarsource.com/browse/SONAR-12140">SONAR-12140</a> for more information
+ */
+public class ProjectsInWarningDaemon implements Startable {
+
+  final static String PROJECTS_IN_WARNING_INTERNAL_PROPERTY = "projectsInWarning";
+
+  private static final Logger LOG = Loggers.get(ProjectsInWarningDaemon.class);
+
+  private static final String FREQUENCY_IN_SECONDS_PROPERTY = "sonar.projectsInWarning.frequencyInSeconds";
+  private static final int DEFAULT_FREQUENCY_IN_SECONDS = 60 * 60 * 24;
+  private static final String THREAD_NAME_PREFIX = "sq-projects-in-warning-service-";
+
+  private static final String LOCK_NAME = "ProjectsInWarn";
+  private static final int LOCK_DURATION_IN_SECOND = 60 * 60;
+
+  private final DbClient dbClient;
+  private final ProjectMeasuresIndex projectMeasuresIndex;
+  private final Configuration config;
+  private final GlobalLockManager lockManager;
+  private final ProjectsInWarning projectsInWarning;
+
+  private ScheduledExecutorService executorService;
+
+  public ProjectsInWarningDaemon(DbClient dbClient, ProjectMeasuresIndex projectMeasuresIndex, Configuration config, GlobalLockManager lockManager,
+    ProjectsInWarning projectsInWarning) {
+    this.dbClient = dbClient;
+    this.projectMeasuresIndex = projectMeasuresIndex;
+    this.config = config;
+    this.lockManager = lockManager;
+    this.projectsInWarning = projectsInWarning;
+  }
+
+  public void notifyStart() {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      Optional<String> internalProperty = dbClient.internalPropertiesDao().selectByKey(dbSession, PROJECTS_IN_WARNING_INTERNAL_PROPERTY);
+      if (internalProperty.isPresent() && internalProperty.get().equals("0")) {
+        projectsInWarning.update(0L);
+        LOG.info("Counting number of projects in warning is not started as there are no projects in this situation.");
+        return;
+      }
+    }
+    LOG.info("Counting number of projects in warning is enabled.");
+    executorService = Executors.newSingleThreadScheduledExecutor(newThreadFactory());
+    executorService.scheduleWithFixedDelay(countProjectsInWarning(), 0, frequency(), TimeUnit.SECONDS);
+  }
+
+  private int frequency() {
+    return config.getInt(FREQUENCY_IN_SECONDS_PROPERTY).orElse(DEFAULT_FREQUENCY_IN_SECONDS);
+  }
+
+  private Runnable countProjectsInWarning() {
+    return () -> {
+      try (DbSession dbSession = dbClient.openSession(false)) {
+        long nbProjectsInWarning = projectMeasuresIndex.search(
+          new ProjectMeasuresQuery()
+            .setQualityGateStatus(WARN)
+            .setIgnoreAuthorization(true),
+          // We only need the number of projects in warning
+          new SearchOptions().setLimit(1)).getTotal();
+        projectsInWarning.update(nbProjectsInWarning);
+        updateProjectsInWarningInDb(dbSession, nbProjectsInWarning);
+        if (nbProjectsInWarning == 0L) {
+          LOG.info("Counting number of projects in warning will be disabled as there are no more projects in warning.");
+          executorService.shutdown();
+        }
+      } catch (Exception e) {
+        LOG.error("Error while counting number of projects in warning: {}", e);
+      }
+    };
+  }
+
+  private void updateProjectsInWarningInDb(DbSession dbSession, long nbProjectsInWarning) {
+    // Only one web node should do the update in db to avoid any collision
+    if (!lockManager.tryLock(LOCK_NAME, LOCK_DURATION_IN_SECOND)) {
+      return;
+    }
+    dbClient.internalPropertiesDao().save(dbSession, PROJECTS_IN_WARNING_INTERNAL_PROPERTY, Long.toString(nbProjectsInWarning));
+    dbSession.commit();
+  }
+
+  @Override
+  public void start() {
+    // Nothing is done here, as this component needs to be started after ES indexing. See PlatformLevelStartup for more info.
+  }
+
+  @Override
+  public void stop() {
+    if (executorService == null) {
+      return;
+    }
+    try {
+      executorService.shutdown();
+      executorService.awaitTermination(5, TimeUnit.SECONDS);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  private static ThreadFactory newThreadFactory() {
+    return new ThreadFactoryBuilder()
+      .setNameFormat(THREAD_NAME_PREFIX + "%d")
+      .setPriority(Thread.MIN_PRIORITY)
+      .build();
+  }
+
+}

server/sonar-server/src/main/java/org/sonar/server/qualitygate/QualityGateModule.java

@@ -63,6 +63,8 @@ public class QualityGateModule extends Module {
       DeleteConditionAction.class,
       UpdateConditionAction.class,
       ProjectStatusAction.class,
-      GetByProjectAction.class);
+      GetByProjectAction.class,
+      ProjectsInWarningDaemon.class,
+      ProjectsInWarning.class);
   }
 }

server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java

@@ -455,6 +455,16 @@ public class ProjectMeasuresIndexTest {
     assertResults(new ProjectMeasuresQuery(), PROJECT1);
   }
 
+  @Test
+  public void return_all_projects_when_setIgnoreAuthorization_is_true() {
+    indexForUser(USER1, newDoc(PROJECT1), newDoc(PROJECT2));
+    indexForUser(USER2, newDoc(PROJECT3));
+    userSession.logIn(USER1);
+
+    assertResults(new ProjectMeasuresQuery().setIgnoreAuthorization(false), PROJECT1, PROJECT2);
+    assertResults(new ProjectMeasuresQuery().setIgnoreAuthorization(true), PROJECT1, PROJECT2, PROJECT3);
+  }
+
   @Test
   public void does_not_return_facet_when_no_facets_in_options() {
     index(

server/sonar-server/src/test/java/org/sonar/server/qualitygate/ProjectsInWarningDaemonTest.java

@@ -0,0 +1,233 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.qualitygate;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.api.measures.CoreMetrics;
+import org.sonar.api.measures.Metric;
+import org.sonar.api.utils.System2;
+import org.sonar.api.utils.log.LogTester;
+import org.sonar.api.utils.log.LoggerLevel;
+import org.sonar.db.DbTester;
+import org.sonar.db.component.ComponentDto;
+import org.sonar.db.metric.MetricDto;
+import org.sonar.server.es.EsTester;
+import org.sonar.server.measure.index.ProjectMeasuresIndex;
+import org.sonar.server.measure.index.ProjectMeasuresIndexer;
+import org.sonar.server.permission.index.PermissionIndexerTester;
+import org.sonar.server.permission.index.WebAuthorizationTypeSupport;
+import org.sonar.server.util.GlobalLockManager;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.sonar.api.measures.Metric.Level.WARN;
+import static org.sonar.db.measure.MeasureTesting.newLiveMeasure;
+import static org.sonar.server.qualitygate.ProjectsInWarningDaemon.PROJECTS_IN_WARNING_INTERNAL_PROPERTY;
+
+public class ProjectsInWarningDaemonTest {
+
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+  @Rule
+  public DbTester db = DbTester.create();
+  @Rule
+  public EsTester es = EsTester.create();
+  @Rule
+  public LogTester logger = new LogTester().setLevel(LoggerLevel.DEBUG);
+
+  private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(es, new ProjectMeasuresIndexer(db.getDbClient(), es.client()));
+  private ProjectMeasuresIndexer projectMeasuresIndexer = new ProjectMeasuresIndexer(db.getDbClient(), es.client());
+  private ProjectMeasuresIndex projectMeasuresIndex = new ProjectMeasuresIndex(es.client(), new WebAuthorizationTypeSupport(null), System2.INSTANCE);
+
+  private MapSettings settings = new MapSettings();
+  private GlobalLockManager lockManager = mock(GlobalLockManager.class);
+  private ProjectsInWarning projectsInWarning = new ProjectsInWarning();
+
+  private ProjectsInWarningDaemon underTest = new ProjectsInWarningDaemon(db.getDbClient(), projectMeasuresIndex, settings.asConfig(), lockManager, projectsInWarning);
+
+  @Before
+  public void setUp() throws Exception {
+    settings.setProperty("sonar.projectsInWarning.frequencyInSeconds", "1");
+  }
+
+  @After
+  public void tearDown() {
+    underTest.stop();
+  }
+
+  @Test
+  public void store_projects_in_warning() throws InterruptedException {
+    allowLockToBeAcquired();
+    MetricDto qualityGateStatus = insertQualityGateStatusMetric();
+    insertProjectInWarning(qualityGateStatus);
+    insertProjectInWarning(qualityGateStatus);
+    // Setting does not exist
+    assertThat(db.getDbClient().internalPropertiesDao().selectByKey(db.getSession(), PROJECTS_IN_WARNING_INTERNAL_PROPERTY)).isEmpty();
+
+    underTest.notifyStart();
+
+    assertProjectsInWarningValue(2L);
+    assertThat(logger.logs(LoggerLevel.INFO)).contains("Counting number of projects in warning is enabled.");
+  }
+
+  @Test
+  public void update_projects_in_warning_when_new_project_in_warning() throws InterruptedException {
+    allowLockToBeAcquired();
+    MetricDto qualityGateStatus = insertQualityGateStatusMetric();
+    ;
+    insertProjectInWarning(qualityGateStatus);
+    insertProjectInWarning(qualityGateStatus);
+    // Setting does not exist
+    assertThat(db.getDbClient().internalPropertiesDao().selectByKey(db.getSession(), PROJECTS_IN_WARNING_INTERNAL_PROPERTY)).isEmpty();
+
+    underTest.notifyStart();
+    // Add a project in warning after the start in order to let the thread do his job
+    insertProjectInWarning(qualityGateStatus);
+
+    assertProjectsInWarningValue(3L);
+    assertThat(logger.logs(LoggerLevel.INFO)).contains("Counting number of projects in warning is enabled.");
+  }
+
+  @Test
+  public void stop_thread_when_number_of_projects_in_warning_reach_zero() throws InterruptedException {
+    allowLockToBeAcquired();
+    MetricDto qualityGateStatus = insertQualityGateStatusMetric();
+    ;
+    ComponentDto project = insertProjectInWarning(qualityGateStatus);
+
+    underTest.notifyStart();
+    assertProjectsInWarningValue(1L);
+    // Set quality gate status of the project to OK => No more projects in warning
+    db.getDbClient().liveMeasureDao().insertOrUpdate(db.getSession(),
+      newLiveMeasure(project, qualityGateStatus).setData(Metric.Level.OK.name()).setValue(null));
+    db.commit();
+    projectMeasuresIndexer.indexOnAnalysis(project.uuid());
+
+    assertProjectsInWarningValue(0L);
+    assertThat(logger.logs(LoggerLevel.INFO))
+      .contains(
+        "Counting number of projects in warning is enabled.",
+        "Counting number of projects in warning will be disabled as there are no more projects in warning.");
+  }
+
+  @Test
+  public void update_internal_properties_when_already_exits_and_projects_in_warnings_more_than_zero() throws InterruptedException {
+    allowLockToBeAcquired();
+    MetricDto qualityGateStatus = insertQualityGateStatusMetric();
+    ;
+    insertProjectInWarning(qualityGateStatus);
+    insertProjectInWarning(qualityGateStatus);
+    // Setting contains 10, it should be updated with new value
+    db.getDbClient().internalPropertiesDao().save(db.getSession(), PROJECTS_IN_WARNING_INTERNAL_PROPERTY, "10");
+    db.commit();
+
+    underTest.notifyStart();
+
+    assertProjectsInWarningValue(2L);
+    assertThat(logger.logs(LoggerLevel.INFO)).contains("Counting number of projects in warning is enabled.");
+  }
+
+  @Test
+  public void store_zero_projects_in_warning_when_no_projects() throws InterruptedException {
+    allowLockToBeAcquired();
+    assertThat(db.getDbClient().internalPropertiesDao().selectByKey(db.getSession(), PROJECTS_IN_WARNING_INTERNAL_PROPERTY)).isEmpty();
+
+    underTest.notifyStart();
+
+    assertProjectsInWarningValue(0L);
+    assertThat(logger.logs(LoggerLevel.INFO)).contains("Counting number of projects in warning is enabled.");
+  }
+
+  @Test
+  public void do_not_compute_projects_in_warning_when_internal_property_is_zero() throws InterruptedException {
+    allowLockToBeAcquired();
+    MetricDto qualityGateStatus = insertQualityGateStatusMetric();
+    ;
+    insertProjectInWarning(qualityGateStatus);
+    // Setting contains 0, even if there are projects in warning it will stay 0 (as it's not possible to have new projects in warning)
+    db.getDbClient().internalPropertiesDao().save(db.getSession(), PROJECTS_IN_WARNING_INTERNAL_PROPERTY, "0");
+    db.commit();
+
+    underTest.notifyStart();
+
+    assertProjectsInWarningValue(0L);
+    assertThat(logger.logs(LoggerLevel.INFO)).contains("Counting number of projects in warning is not started as there are no projects in this situation.");
+  }
+
+  @Test
+  public void do_not_store_projects_in_warning_in_db_when_cannot_acquire_lock() throws InterruptedException {
+    when(lockManager.tryLock(any(), anyInt())).thenReturn(false);
+    MetricDto qualityGateStatus = insertQualityGateStatusMetric();
+    ;
+    insertProjectInWarning(qualityGateStatus);
+
+    underTest.notifyStart();
+
+    waitForValueToBeComputed(1L);
+    assertThat(projectsInWarning.count()).isEqualTo(1L);
+    assertThat(countNumberOfProjectsInWarning()).isEqualTo(0L);
+  }
+
+  private void waitForValueToBeComputed(long expectedValue) throws InterruptedException {
+    for (int i = 0; i < 100; i++) {
+      if (projectsInWarning.isInitialized() && projectsInWarning.count() == expectedValue) {
+        break;
+      }
+      Thread.sleep(100);
+    }
+  }
+
+  private void assertProjectsInWarningValue(long expectedValue) throws InterruptedException {
+    waitForValueToBeComputed(expectedValue);
+    assertThat(projectsInWarning.count()).isEqualTo(expectedValue);
+    assertThat(countNumberOfProjectsInWarning()).isEqualTo(expectedValue);
+  }
+
+  private long countNumberOfProjectsInWarning() {
+    return db.getDbClient().internalPropertiesDao().selectByKey(db.getSession(), PROJECTS_IN_WARNING_INTERNAL_PROPERTY)
+      .map(Long::valueOf)
+      .orElse(0L);
+  }
+
+  private ComponentDto insertProjectInWarning(MetricDto qualityGateStatus) {
+    ComponentDto project = db.components().insertPrivateProject();
+    db.measures().insertLiveMeasure(project, qualityGateStatus, lm -> lm.setData(WARN.name()).setValue(null));
+    authorizationIndexerTester.allowOnlyAnyone(project);
+    projectMeasuresIndexer.indexOnAnalysis(project.uuid());
+    return project;
+  }
+
+  private MetricDto insertQualityGateStatusMetric() {
+    return db.measures().insertMetric(m -> m.setKey(CoreMetrics.ALERT_STATUS_KEY).setValueType(Metric.ValueType.LEVEL.name()));
+  }
+
+  private void allowLockToBeAcquired() {
+    when(lockManager.tryLock(any(), anyInt())).thenReturn(true);
+  }
+
+}

server/sonar-server/src/test/java/org/sonar/server/qualitygate/QualityGateModuleTest.java

@@ -29,6 +29,6 @@ public class QualityGateModuleTest {
   public void verify_count_of_added_components() {
     ComponentContainer container = new ComponentContainer();
     new QualityGateModule().configure(container);
-    assertThat(container.size()).isEqualTo(21 + 2);
+    assertThat(container.size()).isEqualTo(23 + 2);
   }
 }