SONAR-8435 run level3, 4 and startup in background thread in safemode

server/sonar-server/src/main/java/org/sonar/server/app/ProcessCommandWrapper.java

@@ -25,6 +25,11 @@ public interface ProcessCommandWrapper {
    */
   void requestSQRestart();
 
+  /**
+   * Requests to the main process that the WebServer is stopped.
+   */
+  void requestStop();
+
   /**
    * Notifies any listening process that the WebServer is operational.
    */

server/sonar-server/src/main/java/org/sonar/server/app/ProcessCommandWrapperImpl.java

@@ -40,6 +40,11 @@ public class ProcessCommandWrapperImpl implements ProcessCommandWrapper {
     call(VoidMethod.ASK_FOR_RESTART, selfProcessNumber());
   }
 
+  @Override
+  public void requestStop() {
+    call(VoidMethod.ASK_FOR_STOP, selfProcessNumber());
+  }
+
   @Override
   public void notifyOperational() {
     call(VoidMethod.SET_OPERATIONAL, selfProcessNumber());
@@ -70,6 +75,13 @@ public class ProcessCommandWrapperImpl implements ProcessCommandWrapper {
         processCommands.askForRestart();
         return null;
       }
+    },
+    ASK_FOR_STOP() {
+      @Override
+      <T> T callOn(ProcessCommands processCommands) {
+        processCommands.askForStop();
+        return null;
+      }
     };
     abstract <T> T callOn(ProcessCommands processCommands);
   }

server/sonar-server/src/main/java/org/sonar/server/platform/Platform.java

@@ -23,11 +23,14 @@ import com.google.common.collect.Lists;
 import java.util.Collection;
 import java.util.List;
 import java.util.Properties;
+import java.util.function.Supplier;
+import javax.annotation.Nullable;
 import javax.servlet.ServletContext;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.api.utils.log.Profiler;
 import org.sonar.core.platform.ComponentContainer;
+import org.sonar.server.app.ProcessCommandWrapper;
 import org.sonar.server.platform.db.migration.version.DatabaseVersion;
 import org.sonar.server.platform.platformlevel.PlatformLevel;
 import org.sonar.server.platform.platformlevel.PlatformLevel1;
@@ -46,6 +49,8 @@ public class Platform {
 
   private static final Platform INSTANCE = new Platform();
 
+  private final Supplier<PlatformAutoStarter> autoStarterSupplier;
+  private PlatformAutoStarter autoStarter = null;
   private Properties properties;
   private ServletContext servletContext;
   private PlatformLevel level1;
@@ -59,6 +64,17 @@ public class Platform {
   private final List<Object> level4AddedComponents = Lists.newArrayList();
   private final Profiler profiler = Profiler.createIfTrace(Loggers.get(Platform.class));
 
+  private Platform() {
+    this.autoStarterSupplier = () -> {
+      ProcessCommandWrapper processCommandWrapper = getContainer().getComponentByType(ProcessCommandWrapper.class);
+      return new AsynchronousAutoStarter(processCommandWrapper);
+    };
+  }
+
+  protected Platform(Supplier<PlatformAutoStarter> autoStarterSupplier) {
+    this.autoStarterSupplier = autoStarterSupplier;
+  }
+
   public static Platform getInstance() {
     return INSTANCE;
   }
@@ -84,38 +100,50 @@ public class Platform {
       return;
     }
 
-    boolean wentToSafemode = false;
-    if (requireSafeMode()) {
-      LOGGER.info("DB needs migration, entering safe mode");
-      wentToSafemode = true;
-      startSafeModeContainer();
-      currentLevel = levelSafeMode;
-      started = true;
-    }
-    // if AutoDbMigration kicked in, safe mode is not required anymore
-    if (!requireSafeMode()) {
-      if (wentToSafemode) {
-        LOGGER.info("DB has been updated, exiting safe mode");
-      }
-      startLevel34Containers();
-      executeStartupTasks(startup);
-      // switch current container last to avoid giving access to a partially initialized container
-      currentLevel = level4;
-      started = true;
-
-      // stop safemode container if it existed
-      stopSafeModeContainer();
+    boolean dbRequiredMigration = dbRequiresMigration();
+    startSafeModeContainer();
+    currentLevel = levelSafeMode;
+    started = true;
+
+    // if AutoDbMigration kicked in or no DB migration was required, startup can be resumed in another thread
+    if (dbRequiresMigration()) {
+      LOGGER.info("Database needs migration");
+    } else {
+      this.autoStarter = autoStarterSupplier.get();
+      this.autoStarter.execute(() ->  {
+        try {
+          if (dbRequiredMigration) {
+            LOGGER.info("Database has been automatically updated");
+          }
+          startLevel34Containers();
+
+          executeStartupTasks(startup);
+          // switch current container last to avoid giving access to a partially initialized container
+          currentLevel = level4;
+
+          // stop safemode container if it existed
+          stopSafeModeContainer();
+        } catch (Throwable t) {
+          autoStarter.failure(t);
+        } finally {
+          autoStarter.success();
+        }
+      });
     }
   }
 
+  private boolean dbRequiresMigration() {
+    return getDatabaseStatus() != DatabaseVersion.Status.UP_TO_DATE;
+  }
+
   public void restart() {
     restart(Startup.ALL);
   }
 
   protected void restart(Startup startup) {
-
     // switch currentLevel on level1 now to avoid exposing a container in the process of stopping
     currentLevel = level1;
+    autoStarter = null;
 
     // stop containers
     stopSafeModeContainer();
@@ -128,10 +156,6 @@ public class Platform {
     executeStartupTasks(startup);
   }
 
-  private boolean requireSafeMode() {
-    return getDatabaseStatus() != DatabaseVersion.Status.UP_TO_DATE;
-  }
-
   public boolean isStarted() {
     return status() == Status.UP;
   }
@@ -147,7 +171,7 @@ public class Platform {
     PlatformLevel current = this.currentLevel;
     PlatformLevel levelSafe = this.levelSafeMode;
     if (levelSafe != null && current == levelSafe) {
-      return Status.SAFEMODE;
+      return isRunning(this.autoStarter) ? Status.STARTING : Status.SAFEMODE;
     }
     if (current == level4) {
       return Status.UP;
@@ -155,6 +179,10 @@ public class Platform {
     return Status.BOOTING;
   }
 
+  private static boolean isRunning(@Nullable PlatformAutoStarter autoStarter) {
+    return autoStarter != null && autoStarter.isRunning();
+  }
+
   /**
    * Starts level 1
    */
@@ -255,6 +283,7 @@ public class Platform {
       stopSafeModeContainer();
       stopLevel234Containers();
       stopLevel1Container();
+      autoStarter = null;
       currentLevel = null;
       dbConnected = false;
       started = false;
@@ -276,10 +305,63 @@ public class Platform {
   }
 
   public enum Status {
-    BOOTING, SAFEMODE, UP
+    BOOTING, SAFEMODE, STARTING, UP
   }
 
   public enum Startup {
     NO_STARTUP_TASKS, ALL
   }
+
+  public interface PlatformAutoStarter {
+    /**
+     * Let the autostarted execute the provided code.
+     */
+    void execute(Runnable startCode);
+
+    /**
+     * This method is called by executed start code (see {@link #execute(Runnable)} has finished with a failure.
+     */
+    void failure(Throwable t);
+
+    /**
+     * This method is called by executed start code (see {@link #execute(Runnable)} has finished successfully.
+     */
+    void success();
+
+    /**
+     * Indicates whether the AutoStarter is running.
+     */
+    boolean isRunning();
+  }
+
+  private static final class AsynchronousAutoStarter implements PlatformAutoStarter {
+    private final ProcessCommandWrapper processCommandWrapper;
+    private boolean running = true;
+
+    private AsynchronousAutoStarter(ProcessCommandWrapper processCommandWrapper) {
+      this.processCommandWrapper = processCommandWrapper;
+    }
+
+    @Override
+    public void execute(Runnable startCode) {
+      new Thread(startCode, "SQ starter").start();
+    }
+
+    @Override
+    public void failure(Throwable t) {
+      LOGGER.error("Background initialization failed. Stopping SonarQube", t);
+      processCommandWrapper.requestStop();
+      this.running = false;
+    }
+
+    @Override
+    public void success() {
+      this.running = false;
+    }
+
+    @Override
+    public boolean isRunning() {
+      return running;
+    }
+  }
 }

server/sonar-server/src/main/java/org/sonar/server/platform/ws/StatusAction.java

@@ -61,6 +61,8 @@ public class StatusAction implements SystemWsAction {
     controller.createAction("status")
       .setDescription("Get the server status:" +
         "<ul>" +
+        "<li>STARTING: SonarQube Web Server is up and serving some Web Services (eg. api/system/status) " +
+        "but initialization is still ongoing</li>" +
         "<li>UP: SonarQube instance is up and running</li>" +
         "<li>DOWN: SonarQube instance is up but not running because SQ can not connect to database or " +
         "migration has failed (refer to WS /api/system/migrate_db for details) or some other reason (check logs).</li>" +
@@ -100,12 +102,14 @@ public class StatusAction implements SystemWsAction {
     switch (platformStatus) {
       case BOOTING:
         // can not happen since there can not even exist an instance of the current class
-        // unless the Platform's status is UP or SAFEMODE
+        // unless the Platform's status is UP/SAFEMODE/STARTING
         return Status.DOWN;
       case UP:
         return restartFlagHolder.isRestarting() ? Status.RESTARTING : Status.UP;
+      case STARTING:
+        return computeStatusInStarting();
       case SAFEMODE:
-        return computeFromDbMigrationStatus();
+        return computeStatusInSafemode();
       default:
         throw new IllegalArgumentException("Unsupported Platform.Status " + platformStatus);
     }
@@ -120,7 +124,24 @@ public class StatusAction implements SystemWsAction {
     }
   }
 
-  private Status computeFromDbMigrationStatus() {
+  private Status computeStatusInStarting() {
+    DatabaseMigrationState.Status databaseMigrationStatus = migrationState.getStatus();
+    switch (databaseMigrationStatus) {
+      case NONE:
+        return Status.STARTING;
+      case RUNNING:
+        return Status.DB_MIGRATION_RUNNING;
+      case FAILED:
+        return Status.DOWN;
+      case SUCCEEDED:
+        // DB migration can be finished while we haven't yet finished SQ's initialization
+        return Status.DB_MIGRATION_RUNNING;
+      default:
+        throw new IllegalArgumentException("Unsupported DatabaseMigration.Status " + databaseMigrationStatus);
+    }
+  }
+
+  private Status computeStatusInSafemode() {
     DatabaseMigrationState.Status databaseMigrationStatus = migrationState.getStatus();
     switch (databaseMigrationStatus) {
       case NONE:
@@ -130,16 +151,17 @@ public class StatusAction implements SystemWsAction {
       case FAILED:
         return Status.DOWN;
       case SUCCEEDED:
-        // status of Platform is supposed to be UP _before_ DatabaseMigration status becomes UP too
+        // status of Platform will change to STARTING _before_ DatabaseMigration status becomes SUCCEEDED
+        // (see DatabaseMigrationImpl#doDatabaseMigration, platform's restart is requested _before_ DatabaseMigration status is updated)
         // so, in theory, this case can not happen
-        return Status.UP;
+        return Status.DB_MIGRATION_RUNNING;
       default:
         throw new IllegalArgumentException("Unsupported DatabaseMigration.Status " + databaseMigrationStatus);
     }
   }
 
   private enum Status {
-    UP, DOWN, DB_MIGRATION_NEEDED, DB_MIGRATION_RUNNING, RESTARTING
+    UP, DOWN, DB_MIGRATION_NEEDED, DB_MIGRATION_RUNNING, STARTING, RESTARTING
   }
 
 }

server/sonar-server/src/test/java/org/sonar/server/app/ProcessCommandWrapperImplTest.java

@@ -78,6 +78,31 @@ public class ProcessCommandWrapperImplTest {
     }
   }
 
+  @Test
+  public void requestSQStop_throws_IAE_if_process_shared_path_property_not_set() throws Exception {
+    settings.setProperty(PROPERTY_PROCESS_INDEX, 1);
+    ProcessCommandWrapperImpl processCommandWrapper = new ProcessCommandWrapperImpl(settings);
+
+    expectedException.expect(IllegalArgumentException.class);
+    expectedException.expectMessage("Property process.sharedDir is not set");
+
+    processCommandWrapper.requestStop();
+  }
+
+  @Test
+  public void requestSQStop_updates_shareMemory_file() throws IOException {
+    File tmpDir = temp.newFolder().getAbsoluteFile();
+    settings.setProperty(PROPERTY_SHARED_PATH, tmpDir.getAbsolutePath());
+    settings.setProperty(PROPERTY_PROCESS_INDEX, PROCESS_NUMBER);
+
+    ProcessCommandWrapperImpl underTest = new ProcessCommandWrapperImpl(settings);
+    underTest.requestStop();
+
+    try (DefaultProcessCommands processCommands = DefaultProcessCommands.secondary(tmpDir, PROCESS_NUMBER)) {
+      assertThat(processCommands.askedForStop()).isTrue();
+    }
+  }
+
   @Test
   public void notifyOperational_throws_IAE_if_process_sharedDir_property_not_set() throws Exception {
     settings.setProperty(PROPERTY_PROCESS_INDEX, 1);

server/sonar-server/src/test/java/org/sonar/server/platform/ServerTesterPlatform.java

@@ -19,7 +19,13 @@
  */
 package org.sonar.server.platform;
 
+import java.util.function.Supplier;
+
 public class ServerTesterPlatform extends Platform {
+  public ServerTesterPlatform(Supplier<PlatformAutoStarter> autoStarterFactory) {
+    super(autoStarterFactory);
+  }
+
   /**
    * Override to make public
    */

server/sonar-server/src/test/java/org/sonar/server/platform/ws/StatusActionTest.java

@@ -53,13 +53,14 @@ public class StatusActionTest {
   private static final String SERVER_ID = "20150504120436";
   private static final String SERVER_VERSION = "5.1";
   private static final String STATUS_UP = "UP";
+  private static final String STATUS_STARTING = "STARTING";
   private static final String STATUS_DOWN = "DOWN";
   private static final String STATUS_MIGRATION_NEEDED = "DB_MIGRATION_NEEDED";
   private static final String STATUS_MIGRATION_RUNNING = "DB_MIGRATION_RUNNING";
   private static final String STATUS_RESTARTING = "RESTARTING";
   private static final Set<DatabaseMigrationState.Status> SUPPORTED_DATABASE_MIGRATION_STATUSES = of(DatabaseMigrationState.Status.FAILED, DatabaseMigrationState.Status.NONE,
     DatabaseMigrationState.Status.SUCCEEDED, DatabaseMigrationState.Status.RUNNING);
-  private static final Set<Platform.Status> SUPPORTED_PLATFORM_STATUSES = of(Platform.Status.BOOTING, Platform.Status.SAFEMODE, Platform.Status.UP);
+  private static final Set<Platform.Status> SUPPORTED_PLATFORM_STATUSES = of(Platform.Status.BOOTING, Platform.Status.SAFEMODE, Platform.Status.STARTING, Platform.Status.UP);
 
   private static Server server = new Dummy51Server();
   private DatabaseMigrationState migrationState = mock(DatabaseMigrationState.class);
@@ -143,8 +144,8 @@ public class StatusActionTest {
   }
 
   @Test
-  public void status_is_UP_if_platform_is_SAFEMODE_and_databaseMigration_is_SUCCEEDED() throws Exception {
-    verifyStatus(Platform.Status.SAFEMODE, DatabaseMigrationState.Status.SUCCEEDED, STATUS_UP);
+  public void status_is_MIGRATION_RUNNING_if_platform_is_SAFEMODE_and_databaseMigration_is_SUCCEEDED() throws Exception {
+    verifyStatus(Platform.Status.SAFEMODE, DatabaseMigrationState.Status.SUCCEEDED, STATUS_MIGRATION_RUNNING);
   }
 
   @Test
@@ -152,6 +153,26 @@ public class StatusActionTest {
     verifyStatus(Platform.Status.SAFEMODE, DatabaseMigrationState.Status.FAILED, STATUS_DOWN);
   }
 
+  @Test
+  public void status_is_STARTING_if_platform_is_STARTING_and_databaseMigration_is_NONE() throws Exception {
+    verifyStatus(Platform.Status.STARTING, DatabaseMigrationState.Status.NONE, STATUS_STARTING);
+  }
+
+  @Test
+  public void status_is_DB_MIGRATION_RUNNING_if_platform_is_STARTING_and_databaseMigration_is_RUNNING() throws Exception {
+    verifyStatus(Platform.Status.STARTING, DatabaseMigrationState.Status.RUNNING, STATUS_MIGRATION_RUNNING);
+  }
+
+  @Test
+  public void status_is_MIGRATION_RUNNING_if_platform_is_STARTING_and_databaseMigration_is_SUCCEEDED() throws Exception {
+    verifyStatus(Platform.Status.STARTING, DatabaseMigrationState.Status.SUCCEEDED, STATUS_MIGRATION_RUNNING);
+  }
+
+  @Test
+  public void status_is_DOWN_if_platform_is_STARTING_and_databaseMigration_is_FAILED() throws Exception {
+    verifyStatus(Platform.Status.STARTING, DatabaseMigrationState.Status.FAILED, STATUS_DOWN);
+  }
+
   @Test
   public void status_is_DOWN_if_any_error_occurs_when_checking_DB() throws Exception {
     when(isAliveMapper.isAlive()).thenThrow(new RuntimeException("simulated runtime exception when querying DB"));

server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java

@@ -44,6 +44,7 @@ import org.sonar.process.ProcessEntryPoint;
 import org.sonar.process.ProcessProperties;
 import org.sonar.server.es.EsServerHolder;
 import org.sonar.server.platform.BackendCleanup;
+import org.sonar.server.platform.Platform;
 import org.sonar.server.platform.ServerTesterPlatform;
 import org.sonar.server.plugins.UpdateCenterClient;
 import org.sonar.server.ws.WsTester;
@@ -124,7 +125,32 @@ public class ServerTester extends ExternalResource {
       if (!esIndexes) {
         properties.put("sonar.internal.es.disableIndexes", true);
       }
-      platform = new ServerTesterPlatform();
+      platform = new ServerTesterPlatform(() -> new Platform.PlatformAutoStarter() {
+        private boolean running = false;
+
+        @Override
+        public void execute(Runnable startCode) {
+          running = true;
+          startCode.run();
+        }
+
+        @Override
+        public void failure(Throwable t) {
+          stop();
+          Throwables.propagate(t);
+          this.running = false;
+        }
+
+        @Override
+        public void success() {
+          this.running = false;
+        }
+
+        @Override
+        public boolean isRunning() {
+          return this.running;
+        }
+      });
       platform.init(properties, servletContext);
       platform.addComponents(components);
       platform.doStart(startupTasks ? ALL : NO_STARTUP_TASKS);

tests/upgrade/src/test/java/org/sonarsource/sonarqube/upgrade/ServerStatusCall.java

@@ -40,6 +40,7 @@ public class ServerStatusCall extends WsCallAndWait<ServerStatusResponse> {
 
   @Override
   protected boolean shouldWait(ServerStatusResponse serverStatusResponse) {
-    return false;
+    ServerStatusResponse.Status status = serverStatusResponse.getStatus();
+    return status == ServerStatusResponse.Status.STARTING || status == ServerStatusResponse.Status.DB_MIGRATION_RUNNING;
   }
 }

tests/upgrade/src/test/java/org/sonarsource/sonarqube/upgrade/ServerStatusResponse.java

@@ -42,7 +42,7 @@ public class ServerStatusResponse {
     return status;
   }
 
-  public static enum Status {
-    UP, DOWN, DB_MIGRATION_NEEDED, DB_MIGRATION_RUNNING
+  public enum Status {
+    UP, DOWN, STARTING, DB_MIGRATION_NEEDED, DB_MIGRATION_RUNNING
   }
 }

tests/upgrade/src/test/java/org/sonarsource/sonarqube/upgrade/UpgradeTest.java

@@ -54,6 +54,12 @@ public class UpgradeTest {
 
   private static final String PROJECT_KEY = "org.apache.struts:struts-parent";
   private static final String LATEST_JAVA_RELEASE = "LATEST_RELEASE";
+  private static final Version VERSION_5_2 = Version.create("5.2");
+  private static final Version VERSION_5_6_1 = Version.create("5.6.1");
+  private static final Version VERSION_5_6 = Version.create("5.6");
+  private static final Version VERSION_6_0 = Version.create("6.0");
+  private static final Version VERSION_6_1 = Version.create("6.1");
+  private static final Version VERSION_CURRENT = Version.create("DEV");
 
   private Orchestrator orchestrator;
 
@@ -67,22 +73,22 @@ public class UpgradeTest {
 
   @Test
   public void test_upgrade_from_5_6_1() {
-    testDatabaseUpgrade(Version.create("5.6.1"));
+    testDatabaseUpgrade(VERSION_5_6_1);
   }
 
   @Test
   public void test_upgrade_from_5_2_via_5_6() {
-    testDatabaseUpgrade(Version.create("5.2"), Version.create("5.6"));
+    testDatabaseUpgrade(VERSION_5_2, VERSION_5_6);
   }
 
   @Test
   public void test_upgrade_from_6_0() {
-    testDatabaseUpgrade(Version.create("6.0"));
+    testDatabaseUpgrade(VERSION_6_0);
   }
 
   @Test
   public void test_upgrade_from_6_1() {
-    testDatabaseUpgrade(Version.create("6.1"));
+    testDatabaseUpgrade(VERSION_6_1);
   }
 
   private void testDatabaseUpgrade(Version fromVersion, Version... intermediaryVersions) {
@@ -94,13 +100,13 @@ public class UpgradeTest {
 
     Arrays.stream(intermediaryVersions).forEach((sqVersion) -> {
       startOldVersionServer(sqVersion, true);
-      upgrade();
+      upgrade(sqVersion);
       verifyAnalysis(files);
       stopServer();
     });
 
     startDevServer();
-    upgrade();
+    upgrade(VERSION_CURRENT);
     verifyAnalysis(files);
     stopServer();
   }
@@ -112,19 +118,23 @@ public class UpgradeTest {
     browseWebapp();
   }
 
-  private void upgrade() {
-    checkSystemStatus(ServerStatusResponse.Status.DB_MIGRATION_NEEDED);
+  private void upgrade(Version sqVersion) {
+    checkSystemStatus(sqVersion, ServerStatusResponse.Status.DB_MIGRATION_NEEDED);
     checkUrlsBeforeUpgrade();
     ServerMigrationResponse serverMigrationResponse = new ServerMigrationCall(orchestrator).callAndWait();
-    assertThat(serverMigrationResponse.getStatus()).isEqualTo(ServerMigrationResponse.Status.MIGRATION_SUCCEEDED);
-    checkSystemStatus(ServerStatusResponse.Status.UP);
+    assertThat(serverMigrationResponse.getStatus())
+      .describedAs("Migration status of version " + sqVersion + " should be MIGRATION_SUCCEEDED")
+      .isEqualTo(ServerMigrationResponse.Status.MIGRATION_SUCCEEDED);
+    checkSystemStatus(sqVersion, ServerStatusResponse.Status.UP);
     checkUrlsAfterUpgrade();
   }
 
-  private void checkSystemStatus(ServerStatusResponse.Status serverStatus) {
-    ServerStatusResponse serverStatusResponse = new ServerStatusCall(orchestrator).call();
+  private void checkSystemStatus(Version sqVersion, ServerStatusResponse.Status serverStatus) {
+    ServerStatusResponse serverStatusResponse = new ServerStatusCall(orchestrator).callAndWait();
 
-    assertThat(serverStatusResponse.getStatus()).isEqualTo(serverStatus);
+    assertThat(serverStatusResponse.getStatus())
+      .describedAs("Server status of version " + sqVersion + " should be " + serverStatus)
+      .isEqualTo(serverStatus);
   }
 
   private void checkUrlsBeforeUpgrade() {