SONAR-10028 move CeQueueCleaner to Web to support clustering

6 files changed, 2 insertions, 227 deletions

diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java
index 2358876fb0e..19ffc9671e8 100644
--- a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java
@@ -59,7 +59,6 @@ import org.sonar.ce.monitoring.DistributedCEQueueStatusImpl;
 import org.sonar.ce.platform.CECoreExtensionsInstaller;
 import org.sonar.ce.platform.ComputeEngineExtensionInstaller;
 import org.sonar.ce.platform.DatabaseCompatibility;
-import org.sonar.ce.queue.CeQueueCleaner;
 import org.sonar.ce.queue.PurgeCeActivities;
 import org.sonar.ce.task.projectanalysis.ProjectAnalysisTaskModule;
 import org.sonar.ce.task.projectanalysis.analysis.ProjectConfigurationFactory;
@@ -117,7 +116,6 @@ import org.sonar.server.notification.email.EmailNotificationChannel;
 import org.sonar.server.organization.BillingValidationsProxyImpl;
 import org.sonar.server.organization.DefaultOrganizationProviderImpl;
 import org.sonar.server.organization.OrganizationFlagsImpl;
-import org.sonar.server.platform.DefaultServerUpgradeStatus;
 import org.sonar.server.platform.OfficialDistribution;
 import org.sonar.server.platform.ServerFileSystemImpl;
 import org.sonar.server.platform.ServerImpl;
@@ -330,7 +328,6 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer {
 
       // add ReadOnlyPropertiesDao at level2 again so that it shadows PropertiesDao
       ReadOnlyPropertiesDao.class,
-      DefaultServerUpgradeStatus.class,
 
       // plugins
       PluginClassloaderFactory.class,
@@ -485,8 +482,7 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer {
   private static Object[] startupComponents() {
     return new Object[] {
       ServerLifecycleNotifier.class,
-      PurgeCeActivities.class,
-      CeQueueCleaner.class
+      PurgeCeActivities.class
     };
   }
 
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/CeQueueCleaner.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/CeQueueCleaner.java
deleted file mode 100644
index 22e2c6bfc00..00000000000
--- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/CeQueueCleaner.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2019 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.ce.queue;
-
-import java.util.List;
-import org.picocontainer.Startable;
-import org.sonar.api.ce.ComputeEngineSide;
-import org.sonar.api.config.Configuration;
-import org.sonar.api.platform.ServerUpgradeStatus;
-import org.sonar.api.utils.log.Logger;
-import org.sonar.api.utils.log.Loggers;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.process.ProcessProperties;
-
-/**
- * Cleans-up the Compute Engine queue.
- * CE workers must not be started before execution of this class.
- */
-@ComputeEngineSide
-public class CeQueueCleaner implements Startable {
-
-  private static final Logger LOGGER = Loggers.get(CeQueueCleaner.class);
-
-  private final DbClient dbClient;
-  private final ServerUpgradeStatus serverUpgradeStatus;
-  private final InternalCeQueue queue;
-  private final Configuration configuration;
-
-  public CeQueueCleaner(DbClient dbClient, ServerUpgradeStatus serverUpgradeStatus, InternalCeQueue queue, Configuration configuration) {
-    this.dbClient = dbClient;
-    this.serverUpgradeStatus = serverUpgradeStatus;
-    this.queue = queue;
-    this.configuration = configuration;
-  }
-
-  @Override
-  public void start() {
-    if (serverUpgradeStatus.isUpgraded() && !isBlueGreenDeployment()) {
-      cleanOnUpgrade();
-    }
-    cleanUpTaskInputOrphans();
-  }
-
-  private boolean isBlueGreenDeployment() {
-    return configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false);
-  }
-
-  private void cleanOnUpgrade() {
-    // we assume that pending tasks are not compatible with the new version
-    // and can't be processed
-    LOGGER.info("Cancel all pending tasks (due to upgrade)");
-    queue.clear();
-  }
-
-  private void cleanUpTaskInputOrphans() {
-    try (DbSession dbSession = dbClient.openSession(false)) {
-      // Reports that have been processed are not kept in database yet.
-      // They are supposed to be systematically dropped.
-      // Let's clean-up orphans if any.
-      List<String> uuids = dbClient.ceTaskInputDao().selectUuidsNotInQueue(dbSession);
-      dbClient.ceTaskInputDao().deleteByUuids(dbSession, uuids);
-      dbSession.commit();
-    }
-  }
-
-  @Override
-  public void stop() {
-    // nothing to do
-  }
-}
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueue.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueue.java
index c8a06e367b2..44c8561553d 100644
--- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueue.java
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueue.java
@@ -50,16 +50,6 @@ public interface InternalCeQueue extends CeQueue {
   Optional<CeTask> peek(String workerUuid);
 
   /**
-   * Removes all the tasks from the queue, whatever their status. They are marked
-   * as {@link Status#CANCELED} in past activity.
-   * This method can NOT be called when  workers are being executed, as in progress
-   * tasks can't be killed.
-   *
-   * @return the number of canceled tasks
-   */
-  int clear();
-
-  /**
    * Removes a task from the queue and registers it to past activities. This method
    * is called by Compute Engine workers when task is processed and can include an option {@link CeTaskResult} object.
    *
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java
index 4468e1ab7af..13a424cba7d 100644
--- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java
@@ -104,11 +104,6 @@ public class InternalCeQueueImpl extends CeQueueImpl implements InternalCeQueue
   }
 
   @Override
-  public int clear() {
-    return cancelAll(true);
-  }
-
-  @Override
   public void remove(CeTask task, CeActivityDto.Status status, @Nullable CeTaskResult taskResult, @Nullable Throwable error) {
     checkArgument(error == null || status == CeActivityDto.Status.FAILED, "Error can be provided only when status is FAILED");
 
diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java
index e75d0215a3d..68361fe5b64 100644
--- a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java
+++ b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java
@@ -123,7 +123,7 @@ public class ComputeEngineContainerImplTest {
       assertThat(picoContainer.getParent().getParent().getComponentAdapters()).hasSize(
         CONTAINER_ITSELF
           + 25 // MigrationConfigurationModule
-          + 17 // level 2
+          + 16 // level 2
       );
       assertThat(picoContainer.getParent().getParent().getParent().getComponentAdapters()).hasSize(
         COMPONENTS_IN_LEVEL_1_AT_CONSTRUCTION
diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/queue/CeQueueCleanerTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/queue/CeQueueCleanerTest.java
deleted file mode 100644
index 771b13c21a2..00000000000
--- a/server/sonar-ce/src/test/java/org/sonar/ce/queue/CeQueueCleanerTest.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2019 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.ce.queue;
-
-import java.util.Optional;
-import org.apache.commons.io.IOUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.sonar.api.config.internal.MapSettings;
-import org.sonar.api.platform.ServerUpgradeStatus;
-import org.sonar.api.utils.System2;
-import org.sonar.db.DbTester;
-import org.sonar.db.ce.CeQueueDto;
-import org.sonar.db.ce.CeTaskInputDao;
-import org.sonar.db.ce.CeTaskTypes;
-import org.sonar.process.ProcessProperties;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class CeQueueCleanerTest {
-
-  @Rule
-  public DbTester dbTester = DbTester.create(System2.INSTANCE);
-
-  private ServerUpgradeStatus serverUpgradeStatus = mock(ServerUpgradeStatus.class);
-  private InternalCeQueue queue = mock(InternalCeQueue.class);
-  private MapSettings settings = new MapSettings();
-
-  @Test
-  public void start_does_not_reset_in_progress_tasks_to_pending() {
-    insertInQueue("TASK_1", CeQueueDto.Status.PENDING);
-    insertInQueue("TASK_2", CeQueueDto.Status.IN_PROGRESS);
-
-    runCleaner();
-
-    assertThat(dbTester.getDbClient().ceQueueDao().countByStatus(dbTester.getSession(), CeQueueDto.Status.PENDING)).isEqualTo(1);
-    assertThat(dbTester.getDbClient().ceQueueDao().countByStatus(dbTester.getSession(), CeQueueDto.Status.IN_PROGRESS)).isEqualTo(1);
-  }
-
-  @Test
-  public void start_clears_queue_if_version_upgrade() {
-    when(serverUpgradeStatus.isUpgraded()).thenReturn(true);
-
-    runCleaner();
-
-    verify(queue).clear();
-  }
-
-  @Test
-  public void start_does_not_clear_queue_if_version_upgrade_but_blue_green_deployment() {
-    when(serverUpgradeStatus.isUpgraded()).thenReturn(true);
-    settings.setProperty(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey(), true);
-
-    runCleaner();
-
-    verify(queue, never()).clear();
-  }
-
-  @Test
-  public void start_deletes_orphan_report_files() {
-    // analysis reports are persisted but the associated
-    // task is not in the queue
-    insertInQueue("TASK_1", CeQueueDto.Status.PENDING);
-    insertTaskData("TASK_1");
-    insertTaskData("TASK_2");
-
-    runCleaner();
-
-    CeTaskInputDao dataDao = dbTester.getDbClient().ceTaskInputDao();
-    Optional<CeTaskInputDao.DataStream> task1Data = dataDao.selectData(dbTester.getSession(), "TASK_1");
-    assertThat(task1Data).isPresent();
-    task1Data.get().close();
-
-    assertThat(dataDao.selectData(dbTester.getSession(), "TASK_2")).isNotPresent();
-  }
-
-  private CeQueueDto insertInQueue(String taskUuid, CeQueueDto.Status status) {
-    CeQueueDto dto = new CeQueueDto();
-    dto.setTaskType(CeTaskTypes.REPORT);
-    dto.setComponentUuid("PROJECT_1");
-    dto.setUuid(taskUuid);
-    dto.setStatus(status);
-    dbTester.getDbClient().ceQueueDao().insert(dbTester.getSession(), dto);
-    dbTester.getSession().commit();
-    return dto;
-  }
-
-  private void insertTaskData(String taskUuid) {
-    dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), taskUuid, IOUtils.toInputStream("{binary}"));
-    dbTester.getSession().commit();
-  }
-
-  private void runCleaner() {
-    CeQueueCleaner cleaner = new CeQueueCleaner(dbTester.getDbClient(), serverUpgradeStatus, queue, settings.asConfig());
-    cleaner.start();
-  }
-}