SONAR-12043 main process supports graceful and hard stop

12 files changed, 424 insertions, 65 deletions

diff --git a/server/sonar-main/src/main/java/org/sonar/application/NodeLifecycle.java b/server/sonar-main/src/main/java/org/sonar/application/NodeLifecycle.java
index 4be33af9458..fe9830a8dd6 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/NodeLifecycle.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/NodeLifecycle.java
@@ -33,6 +33,7 @@ import static org.sonar.application.NodeLifecycle.State.OPERATIONAL;
 import static org.sonar.application.NodeLifecycle.State.RESTARTING;
 import static org.sonar.application.NodeLifecycle.State.STARTING;
 import static org.sonar.application.NodeLifecycle.State.STOPPED;
+import static org.sonar.application.NodeLifecycle.State.HARD_STOPPING;
 import static org.sonar.application.NodeLifecycle.State.STOPPING;
 
 /**
@@ -55,9 +56,12 @@ class NodeLifecycle {
     // at least one process is still stopping as part of a node restart
     RESTARTING,
 
-    // at least one process is still stopping
+    // at least one process is still stopping as part of a node graceful stop
     STOPPING,
 
+    // at least one process is still stopping as part of a node hard stop
+    HARD_STOPPING,
+
     // all processes are stopped
     STOPPED
   }
@@ -69,10 +73,11 @@ class NodeLifecycle {
   private static Map<State, Set<State>> buildTransitions() {
     Map<State, Set<State>> res = new EnumMap<>(State.class);
     res.put(INIT, toSet(STARTING));
-    res.put(STARTING, toSet(OPERATIONAL, RESTARTING, STOPPING, STOPPED));
-    res.put(OPERATIONAL, toSet(RESTARTING, STOPPING, STOPPED));
-    res.put(RESTARTING, toSet(STOPPING, STOPPED));
-    res.put(STOPPING, toSet(STOPPED));
+    res.put(STARTING, toSet(OPERATIONAL, RESTARTING, HARD_STOPPING, STOPPED));
+    res.put(OPERATIONAL, toSet(RESTARTING, STOPPING, HARD_STOPPING, STOPPED));
+    res.put(STOPPING, toSet(HARD_STOPPING, STOPPED));
+    res.put(RESTARTING, toSet(STARTING, HARD_STOPPING, STOPPED));
+    res.put(HARD_STOPPING, toSet(STOPPED));
     res.put(STOPPED, toSet(STARTING));
     return Collections.unmodifiableMap(res);
   }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/Scheduler.java b/server/sonar-main/src/main/java/org/sonar/application/Scheduler.java
index b01448b4787..39ca8b4d3e9 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/Scheduler.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/Scheduler.java
@@ -24,6 +24,11 @@ public interface Scheduler {
   void schedule() throws InterruptedException;
 
   /**
+   * Gracefully stops all processes and waits for them to be down.
+   */
+  void stop();
+
+  /**
    * Stops all processes and waits for them to be down.
    */
   void hardStop();
diff --git a/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java b/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java
index b218afc3ff1..e6e601d2b04 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
+import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.application.command.AbstractCommand;
@@ -38,11 +39,16 @@ import org.sonar.application.process.ManagedProcessLifecycle;
 import org.sonar.application.process.ProcessLifecycleListener;
 import org.sonar.process.ProcessId;
 
+import static org.sonar.application.NodeLifecycle.State.HARD_STOPPING;
+import static org.sonar.application.NodeLifecycle.State.RESTARTING;
+import static org.sonar.application.NodeLifecycle.State.STOPPED;
+import static org.sonar.application.NodeLifecycle.State.STOPPING;
 import static org.sonar.application.process.ManagedProcessHandler.Timeout.newTimeout;
 
 public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, ProcessLifecycleListener, AppStateListener {
 
   private static final Logger LOG = LoggerFactory.getLogger(SchedulerImpl.class);
+  private static final ManagedProcessHandler.Timeout HARD_STOP_TIMEOUT = newTimeout(1, TimeUnit.MINUTES);
 
   private final AppSettings settings;
   private final AppReloader appReloader;
@@ -56,6 +62,7 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
   private final EnumMap<ProcessId, ManagedProcessHandler> processesById = new EnumMap<>(ProcessId.class);
   private final AtomicInteger operationalCountDown = new AtomicInteger();
   private final AtomicInteger stopCountDown = new AtomicInteger(0);
+  private RestartStopperThread restartStopperThread;
   private HardStopperThread hardStopperThread;
   private RestarterThread restarterThread;
   private long processWatcherDelayMs = ManagedProcessHandler.DEFAULT_WATCHER_DELAY_MS;
@@ -87,8 +94,8 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
         .addProcessLifecycleListener(this)
         .addEventListener(this)
         .setWatcherDelayMs(processWatcherDelayMs)
-        // FIXME MMF-1673 timeout here must be changed to sonar.ce.task.timeout + 5 minutes if CE
-        .setHardStopTimeout(newTimeout(1, TimeUnit.MINUTES))
+        .setStopTimeout(stopTimeoutFor(processId))
+        .setHardStopTimeout(HARD_STOP_TIMEOUT)
         .build();
       processesById.put(process.getProcessId(), process);
     }
@@ -97,6 +104,21 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
     tryToStartAll();
   }
 
+  private static ManagedProcessHandler.Timeout stopTimeoutFor(ProcessId processId) {
+    switch (processId) {
+      case ELASTICSEARCH:
+        return HARD_STOP_TIMEOUT;
+      case WEB_SERVER:
+        return newTimeout(10, TimeUnit.MINUTES);
+      case COMPUTE_ENGINE:
+        // FIXME MMF-1673 make compute engine timeout configurable for ITs
+        return newTimeout(6, TimeUnit.HOURS);
+      case APP:
+      default:
+        throw new IllegalArgumentException("Unsupported processId " + processId);
+    }
+  }
+
   private void tryToStartAll() throws InterruptedException {
     tryToStartEs();
     tryToStartWeb();
@@ -155,9 +177,9 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
 
     try {
       processHandler.start(() -> {
-      AbstractCommand command = commandSupplier.get();
-      return processLauncher.launch(command);
-    });
+        AbstractCommand command = commandSupplier.get();
+        return processLauncher.launch(command);
+      });
     } catch (RuntimeException e) {
       // failed to start command -> stop everything
       hardStop();
@@ -165,23 +187,42 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
     }
   }
 
-  private void hardStopAll() throws InterruptedException {
+  @Override
+  public void stop() {
+    if (nodeLifecycle.tryToMoveTo(STOPPING)) {
+      LOG.info("Stopping SonarQube");
+    }
+    stopImpl();
+  }
+
+  private void stopImpl() {
+    try {
+      stopAll();
+      finalizeStop();
+    } catch (InterruptedException e) {
+      LOG.debug("Stop interrupted", e);
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  private void stopAll() throws InterruptedException {
     // order is important for non-cluster mode
-    hardStopProcess(ProcessId.COMPUTE_ENGINE);
-    hardStopProcess(ProcessId.WEB_SERVER);
-    hardStopProcess(ProcessId.ELASTICSEARCH);
+    stopProcess(ProcessId.COMPUTE_ENGINE);
+    stopProcess(ProcessId.WEB_SERVER);
+    stopProcess(ProcessId.ELASTICSEARCH);
   }
 
   /**
-   * Request for quick stop then blocks until process is stopped.
+   * Request for graceful stop then blocks until process is stopped.
    * Returns immediately if the process is disabled in configuration.
    *
    * @throws InterruptedException if {@link ManagedProcessHandler#hardStop()} throws a {@link InterruptedException}
    */
-  private void hardStopProcess(ProcessId processId) throws InterruptedException {
+  private void stopProcess(ProcessId processId) throws InterruptedException {
     ManagedProcessHandler process = processesById.get(processId);
     if (process != null) {
-      process.hardStop();
+      LOG.debug("Stopping [{}]...", process.getProcessId().getKey());
+      process.stop();
     }
   }
 
@@ -190,17 +231,16 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
    */
   @Override
   public void hardStop() {
-    if (nodeLifecycle.tryToMoveTo(NodeLifecycle.State.STOPPING)) {
-      LOG.info("Stopping SonarQube");
+    if (nodeLifecycle.tryToMoveTo(HARD_STOPPING)) {
+      LOG.info("Hard stopping SonarQube");
     }
+    hardStopImpl();
+  }
+
+  private void hardStopImpl() {
     try {
       hardStopAll();
-      if (hardStopperThread != null && Thread.currentThread() != hardStopperThread) {
-        hardStopperThread.interrupt();
-      }
-      if (restarterThread != null && Thread.currentThread() != restarterThread) {
-        restarterThread.interrupt();
-      }
+      finalizeStop();
     } catch (InterruptedException e) {
       // ignore and assume SQ stop is handled by another thread
       LOG.debug("Stopping all processes was interrupted in the middle of a hard stop" +
@@ -210,6 +250,49 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
     awaitTermination.countDown();
   }
 
+  private void hardStopAll() throws InterruptedException {
+    // order is important for non-cluster mode
+    hardStopProcess(ProcessId.COMPUTE_ENGINE);
+    hardStopProcess(ProcessId.WEB_SERVER);
+    hardStopProcess(ProcessId.ELASTICSEARCH);
+
+    // if all process are already stopped (may occur, eg., when stopping because restart of 1st process failed),
+    // node state won't be updated on process stopped callback, so we must ensure
+    // the node's state is updated
+    if (nodeLifecycle.getState() != RESTARTING) {
+      nodeLifecycle.tryToMoveTo(STOPPED);
+    }
+  }
+
+  private void finalizeStop() {
+    if (nodeLifecycle.getState() == STOPPED) {
+      interrupt(restartStopperThread);
+      interrupt(hardStopperThread);
+      interrupt(restarterThread);
+    }
+  }
+
+  private static void interrupt(@Nullable Thread thread) {
+    if (thread != null
+      // do not interrupt oneself
+      && Thread.currentThread() != thread) {
+      thread.interrupt();
+    }
+  }
+
+  /**
+   * Request for graceful stop then blocks until process is stopped.
+   * Returns immediately if the process is disabled in configuration.
+   *
+   * @throws InterruptedException if {@link ManagedProcessHandler#hardStop()} throws a {@link InterruptedException}
+   */
+  private void hardStopProcess(ProcessId processId) throws InterruptedException {
+    ManagedProcessHandler process = processesById.get(processId);
+    if (process != null) {
+      process.hardStop();
+    }
+  }
+
   @Override
   public void awaitTermination() {
     try {
@@ -223,16 +306,17 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
   public void onManagedProcessEvent(ProcessId processId, Type type) {
     if (type == Type.OPERATIONAL) {
       onProcessOperational(processId);
-    } else if (type == Type.ASK_FOR_RESTART && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.RESTARTING)) {
+    } else if (type == Type.ASK_FOR_RESTART && nodeLifecycle.tryToMoveTo(RESTARTING)) {
       LOG.info("SQ restart requested by Process[{}]", processId.getKey());
-      hardStopAsync();
+      stopAsyncForRestart();
     }
   }
 
   private void onProcessOperational(ProcessId processId) {
     LOG.info("Process[{}] is up", processId.getKey());
     appState.setOperational(processId);
-    if (operationalCountDown.decrementAndGet() == 0 && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.OPERATIONAL)) {
+    boolean lastProcessStarted = operationalCountDown.decrementAndGet() == 0;
+    if (lastProcessStarted && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.OPERATIONAL)) {
       LOG.info("SonarQube is up");
     }
   }
@@ -271,13 +355,14 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
     boolean lastProcessStopped = stopCountDown.decrementAndGet() == 0;
     switch (nodeLifecycle.getState()) {
       case RESTARTING:
-        if (lastProcessStopped && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.STOPPED)) {
+        if (lastProcessStopped) {
           LOG.info("SonarQube is restarting");
           restartAsync();
         }
         break;
-      case STOPPING:
-        if (lastProcessStopped && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.STOPPED)) {
+      case HARD_STOPPING:
+        if (lastProcessStopped && nodeLifecycle.tryToMoveTo(STOPPED)) {
+          LOG.info("SonarQube is stopped");
           // all processes are stopped, no restart requested
           // Let's clean-up resources
           hardStop();
@@ -290,11 +375,31 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
   }
 
   private void hardStopAsync() {
+    if (hardStopperThread != null) {
+      LOG.debug("Hard stopper thread was not null (name is \"{}\")", hardStopperThread.getName(), new Exception());
+      hardStopperThread.interrupt();
+    }
+
     hardStopperThread = new HardStopperThread();
     hardStopperThread.start();
   }
 
+  private void stopAsyncForRestart() {
+    if (restartStopperThread != null) {
+      LOG.debug("Restart stopper thread was not null", new Exception());
+      restartStopperThread.interrupt();
+    }
+
+    restartStopperThread = new RestartStopperThread();
+    restartStopperThread.start();
+  }
+
   private void restartAsync() {
+    if (restarterThread != null) {
+      LOG.debug("Restarter thread was not null (name is \"{}\")", restarterThread.getName(), new Exception());
+      restarterThread.interrupt();
+    }
+
     restarterThread = new RestarterThread();
     restarterThread.start();
   }
@@ -314,24 +419,33 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
         LOG.debug("{} thread was interrupted", getName(), e);
         super.interrupt();
       } catch (Exception e) {
-        LOG.error("Fail to restart", e);
+        LOG.error("Failed to restart", e);
         hardStop();
       }
     }
   }
 
+  private class RestartStopperThread extends Thread {
+    public RestartStopperThread() {
+      super("Restart stopper");
+    }
+
+    @Override
+    public void run() {
+      stopImpl();
+    }
+  }
+
   private class HardStopperThread extends Thread {
+
     public HardStopperThread() {
       super("Hard stopper");
     }
 
     @Override
     public void run() {
-      try {
-        hardStopAll();
-      } catch (InterruptedException e) {
-        LOG.debug("{} thread was interrupted", getName(), e);
-        interrupt();
+      if (nodeLifecycle.tryToMoveTo(HARD_STOPPING)) {
+        hardStopImpl();
       }
     }
   }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
index c8e0badbe77..fd028f5cd10 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
@@ -115,6 +115,11 @@ public class EsManagedProcess extends AbstractManagedProcess {
   }
 
   @Override
+  public void askForStop() {
+    askForHardStop();
+  }
+
+  @Override
   public void askForHardStop() {
     process.destroy();
   }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcess.java
index 9cb0028e0a1..14eccb184b6 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcess.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcess.java
@@ -67,6 +67,8 @@ public interface ManagedProcess {
    */
   boolean isOperational();
 
+  void askForStop();
+
   /**
    * Send request to quick stop to the process
    */
@@ -84,5 +86,4 @@ public interface ManagedProcess {
    * Child process will typically stop sending the signal requesting restart from now on.
    */
   void acknowledgeAskForRestart();
-
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessHandler.java b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessHandler.java
index 791bc351ae4..1104833d6fd 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessHandler.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessHandler.java
@@ -40,6 +40,7 @@ public class ManagedProcessHandler {
   private final ProcessId processId;
   private final ManagedProcessLifecycle lifecycle;
   private final List<ManagedProcessEventListener> eventListeners;
+  private final Timeout stopTimeout;
   private final Timeout hardStopTimeout;
   private final long watcherDelayMs;
 
@@ -56,6 +57,7 @@ public class ManagedProcessHandler {
     this.processId = requireNonNull(builder.processId, "processId can't be null");
     this.lifecycle = new ManagedProcessLifecycle(this.processId, builder.lifecycleListeners);
     this.eventListeners = builder.eventListeners;
+    this.stopTimeout = builder.stopTimeout;
     this.hardStopTimeout = builder.hardStopTimeout;
     this.watcherDelayMs = builder.watcherDelayMs;
     this.stopWatcher = new StopWatcher();
@@ -94,6 +96,20 @@ public class ManagedProcessHandler {
     return lifecycle.getState();
   }
 
+  public void stop() throws InterruptedException {
+    if (lifecycle.tryToMoveTo(ManagedProcessLifecycle.State.STOPPING)) {
+      stopImpl();
+      if (process != null && process.isAlive()) {
+        LOG.info("{} failed to stop in a graceful fashion. Hard stopping it.", processId.getKey());
+      }
+      // enforce stop and clean-up even if process has been quickly stopped
+      stopForcibly();
+    } else {
+      // already stopping or stopped
+      waitForDown();
+    }
+  }
+
   /**
    * Sends kill signal and awaits termination. No guarantee that process is gracefully terminated (=shutdown hooks
    * executed). It depends on OS.
@@ -123,6 +139,21 @@ public class ManagedProcessHandler {
     }
   }
 
+  private void stopImpl() throws InterruptedException {
+    if (process == null) {
+      return;
+    }
+    try {
+      process.askForStop();
+      process.waitFor(stopTimeout.getDuration(), stopTimeout.getUnit());
+    } catch (InterruptedException e) {
+      // can't wait for the termination of process. Let's assume it's down.
+      throw rethrowWithWarn(e, format("Interrupted while stopping process %s", processId));
+    } catch (Throwable e) {
+      LOG.error("Failed asking for graceful stop of process {}", processId, e);
+    }
+  }
+
   private void hardStopImpl() throws InterruptedException {
     if (process == null) {
       return;
@@ -132,15 +163,18 @@ public class ManagedProcessHandler {
       process.waitFor(hardStopTimeout.getDuration(), hardStopTimeout.getUnit());
     } catch (InterruptedException e) {
       // can't wait for the termination of process. Let's assume it's down.
-      String errorMessage = format("Interrupted while hard stopping process %s", processId);
-      LOG.warn(errorMessage, e);
-      Thread.currentThread().interrupt();
-      throw new InterruptedException(errorMessage);
+      throw rethrowWithWarn(e, format("Interrupted while hard stopping process %s", processId));
     } catch (Throwable e) {
       LOG.error("Failed while asking for hard stop of process {}", processId, e);
     }
   }
 
+  private static InterruptedException rethrowWithWarn(InterruptedException e, String errorMessage) {
+    LOG.warn(errorMessage, e);
+    Thread.currentThread().interrupt();
+    return new InterruptedException(errorMessage);
+  }
+
   public void stopForcibly() {
     eventWatcher.interrupt();
     stopWatcher.interrupt();
@@ -242,7 +276,8 @@ public class ManagedProcessHandler {
     private final List<ManagedProcessEventListener> eventListeners = new ArrayList<>();
     private final List<ProcessLifecycleListener> lifecycleListeners = new ArrayList<>();
     private long watcherDelayMs = DEFAULT_WATCHER_DELAY_MS;
-    private Timeout hardStopTimeout = new Timeout(1, TimeUnit.MINUTES);
+    private Timeout stopTimeout;
+    private Timeout hardStopTimeout;
 
     private Builder(ProcessId processId) {
       this.processId = processId;
@@ -266,12 +301,27 @@ public class ManagedProcessHandler {
       return this;
     }
 
+    public Builder setStopTimeout(Timeout stopTimeout) {
+      this.stopTimeout = ensureStopTimeoutNonNull(stopTimeout);
+      return this;
+    }
+
     public Builder setHardStopTimeout(Timeout hardStopTimeout) {
-      this.hardStopTimeout = requireNonNull(hardStopTimeout, "hardStopTimeout can't be null");
+      this.hardStopTimeout = ensureHardStopTimeoutNonNull(hardStopTimeout);
       return this;
     }
 
+    private static Timeout ensureStopTimeoutNonNull(Timeout stopTimeout) {
+      return requireNonNull(stopTimeout, "stopTimeout can't be null");
+    }
+
+    private static Timeout ensureHardStopTimeoutNonNull(Timeout hardStopTimeout) {
+      return requireNonNull(hardStopTimeout, "hardStopTimeout can't be null");
+    }
+
     public ManagedProcessHandler build() {
+      ensureStopTimeoutNonNull(this.stopTimeout);
+      ensureHardStopTimeoutNonNull(this.hardStopTimeout);
       return new ManagedProcessHandler(this);
     }
   }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessLifecycle.java b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessLifecycle.java
index dc103b4f8db..9c4ea29a9a0 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessLifecycle.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessLifecycle.java
@@ -30,16 +30,17 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.process.ProcessId;
 
+import static org.sonar.application.process.ManagedProcessLifecycle.State.HARD_STOPPING;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.INIT;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STARTED;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STARTING;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STOPPED;
-import static org.sonar.application.process.ManagedProcessLifecycle.State.HARD_STOPPING;
+import static org.sonar.application.process.ManagedProcessLifecycle.State.STOPPING;
 
 public class ManagedProcessLifecycle {
 
   public enum State {
-    INIT, STARTING, STARTED, HARD_STOPPING, STOPPED
+    INIT, STARTING, STARTED, STOPPING, HARD_STOPPING, STOPPED
   }
 
   private static final Logger LOG = LoggerFactory.getLogger(ManagedProcessLifecycle.class);
@@ -62,8 +63,9 @@ public class ManagedProcessLifecycle {
   private static Map<State, Set<State>> buildTransitions() {
     Map<State, Set<State>> res = new EnumMap<>(State.class);
     res.put(INIT, toSet(STARTING));
-    res.put(STARTING, toSet(STARTED, HARD_STOPPING, STOPPED));
-    res.put(STARTED, toSet(HARD_STOPPING, STOPPED));
+    res.put(STARTING, toSet(STARTED, STOPPING, HARD_STOPPING, STOPPED));
+    res.put(STARTED, toSet(STOPPING, HARD_STOPPING, STOPPED));
+    res.put(STOPPING, toSet(HARD_STOPPING, STOPPED));
     res.put(HARD_STOPPING, toSet(STOPPED));
     res.put(STOPPED, toSet());
     return Collections.unmodifiableMap(res);
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessCommandsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessCommandsManagedProcess.java
index 975d746193c..4ce128bd56f 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessCommandsManagedProcess.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessCommandsManagedProcess.java
@@ -42,6 +42,14 @@ public class ProcessCommandsManagedProcess extends AbstractManagedProcess {
   }
 
   /**
+   * Send request to gracefully stop to the process (via ipc shared memory)
+   */
+  @Override
+  public void askForStop() {
+    commands.askForStop();
+  }
+
+  /**
    * Send request to quickly stop to the process (via ipc shared memory)
    */
   @Override
diff --git a/server/sonar-main/src/test/java/org/sonar/application/NodeLifecycleTest.java b/server/sonar-main/src/test/java/org/sonar/application/NodeLifecycleTest.java
new file mode 100644
index 00000000000..5ebfd9587fd
--- /dev/null
+++ b/server/sonar-main/src/test/java/org/sonar/application/NodeLifecycleTest.java
@@ -0,0 +1,106 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.sonar.application.NodeLifecycle.State.HARD_STOPPING;
+import static org.sonar.application.NodeLifecycle.State.INIT;
+import static org.sonar.application.NodeLifecycle.State.OPERATIONAL;
+import static org.sonar.application.NodeLifecycle.State.RESTARTING;
+import static org.sonar.application.NodeLifecycle.State.STARTING;
+import static org.sonar.application.NodeLifecycle.State.STOPPED;
+import static org.sonar.application.NodeLifecycle.State.STOPPING;
+
+public class NodeLifecycleTest {
+  private NodeLifecycle underTest = new NodeLifecycle();
+
+  @Test
+  public void verify_regular_start_and_graceful_stop_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPING);
+    assertThat(underTest.tryToMoveTo(STOPPED)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPED);
+  }
+
+  @Test
+  public void verify_start_and_hard_stop_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(HARD_STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(HARD_STOPPING);
+    assertThat(underTest.tryToMoveTo(STOPPED)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPED);
+  }
+
+  @Test
+  public void verify_start_graceful_stop_interrupted_by_hard_stop_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPING);
+    assertThat(underTest.tryToMoveTo(HARD_STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(HARD_STOPPING);
+    assertThat(underTest.tryToMoveTo(STOPPED)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPED);
+  }
+
+  @Test
+  public void verify_regular_start_restart_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(RESTARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(RESTARTING);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+  }
+
+  @Test
+  public void verify_failed_restart_resulting_in_hard_stop_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(RESTARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(RESTARTING);
+    assertThat(underTest.tryToMoveTo(HARD_STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(HARD_STOPPING);
+    assertThat(underTest.tryToMoveTo(STOPPED)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPED);
+  }
+}
diff --git a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java
index 36e5030c6f7..533e9abc932 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java
@@ -127,6 +127,44 @@ public class SchedulerImplTest {
     processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isTrue());
 
     // processes are stopped in reverse order of startup
+    underTest.stop();
+    assertThat(orderedStops).containsExactly(COMPUTE_ENGINE, WEB_SERVER, ELASTICSEARCH);
+    processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse());
+
+    // does nothing because scheduler is already terminated
+    underTest.awaitTermination();
+  }
+
+  @Test
+  public void start_and_hard_stop_sequence_of_ES_WEB_CE_in_order() throws Exception {
+    SchedulerImpl underTest = newScheduler(false);
+    underTest.schedule();
+
+    // elasticsearch does not have preconditions to start
+    TestManagedProcess es = processLauncher.waitForProcess(ELASTICSEARCH);
+    assertThat(es.isAlive()).isTrue();
+    assertThat(processLauncher.processes).hasSize(1);
+
+    // elasticsearch becomes operational -> web leader is starting
+    es.operational = true;
+    waitForAppStateOperational(appState, ELASTICSEARCH);
+    TestManagedProcess web = processLauncher.waitForProcess(WEB_SERVER);
+    assertThat(web.isAlive()).isTrue();
+    assertThat(processLauncher.processes).hasSize(2);
+    assertThat(processLauncher.commands).containsExactly(esScriptCommand, webLeaderCommand);
+
+    // web becomes operational -> CE is starting
+    web.operational = true;
+    waitForAppStateOperational(appState, WEB_SERVER);
+    TestManagedProcess ce = processLauncher.waitForProcess(COMPUTE_ENGINE);
+    assertThat(ce.isAlive()).isTrue();
+    assertThat(processLauncher.processes).hasSize(3);
+    assertThat(processLauncher.commands).containsExactly(esScriptCommand, webLeaderCommand, ceCommand);
+
+    // all processes are up
+    processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isTrue());
+
+    // processes are stopped in reverse order of startup
     underTest.hardStop();
     assertThat(orderedStops).containsExactly(COMPUTE_ENGINE, WEB_SERVER, ELASTICSEARCH);
     processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse());
@@ -353,7 +391,7 @@ public class SchedulerImplTest {
     private ManagedProcess launchImpl(AbstractCommand<?> javaCommand) {
       commands.add(javaCommand);
       if (makeStartupFail == javaCommand.getProcessId()) {
-        throw new IllegalStateException("cannot start " + javaCommand.getProcessId());
+        throw new IllegalStateException("Faking startup of java command failing for " + javaCommand.getProcessId());
       }
       TestManagedProcess process = new TestManagedProcess(javaCommand.getProcessId());
       processes.put(javaCommand.getProcessId(), process);
@@ -402,6 +440,7 @@ public class SchedulerImplTest {
     private final ProcessId processId;
     private final CountDownLatch alive = new CountDownLatch(1);
     private boolean operational = false;
+    private boolean askedForStop = false;
     private boolean askedForRestart = false;
 
     private TestManagedProcess(ProcessId processId) {
@@ -428,6 +467,12 @@ public class SchedulerImplTest {
     }
 
     @Override
+    public void askForStop() {
+      this.askedForStop = true;
+      destroyForcibly();
+    }
+
+    @Override
     public void askForHardStop() {
       destroyForcibly();
     }
diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessHandlerTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessHandlerTest.java
index 882ac36a1f9..ae395897423 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessHandlerTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessHandlerTest.java
@@ -49,7 +49,7 @@ public class ManagedProcessHandlerTest {
 
   @Test
   public void initial_state_is_INIT() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
 
     assertThat(underTest.getProcessId()).isEqualTo(A_PROCESS_ID);
     assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.INIT);
@@ -58,7 +58,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void start_and_stop_process() {
     ProcessLifecycleListener listener = mock(ProcessLifecycleListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addProcessLifecycleListener(listener)
       .build();
 
@@ -79,34 +79,44 @@ public class ManagedProcessHandlerTest {
     }
   }
 
+  public ManagedProcessHandler.Builder newHanderBuilder(ProcessId aProcessId) {
+    return ManagedProcessHandler.builder(aProcessId)
+      .setStopTimeout(newTimeout(1, TimeUnit.SECONDS))
+      .setHardStopTimeout(newTimeout(1, TimeUnit.SECONDS));
+  }
+
   @Test
   public void start_does_not_nothing_if_already_started_once() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
 
     try (TestManagedProcess testProcess = new TestManagedProcess()) {
       assertThat(underTest.start(() -> testProcess)).isTrue();
       assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.STARTED);
 
-      assertThat(underTest.start(() -> {throw new IllegalStateException();})).isFalse();
+      assertThat(underTest.start(() -> {
+        throw new IllegalStateException();
+      })).isFalse();
       assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.STARTED);
     }
   }
 
   @Test
   public void start_throws_exception_and_move_to_state_STOPPED_if_execution_of_command_fails() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
 
     expectedException.expect(IllegalStateException.class);
     expectedException.expectMessage("error");
 
-    underTest.start(() -> {throw new IllegalStateException("error");});
+    underTest.start(() -> {
+      throw new IllegalStateException("error");
+    });
     assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.STOPPED);
   }
 
   @Test
   public void send_event_when_process_is_operational() {
     ManagedProcessEventListener listener = mock(ManagedProcessEventListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addEventListener(listener)
       .build();
 
@@ -124,7 +134,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void operational_event_is_sent_once() {
     ManagedProcessEventListener listener = mock(ManagedProcessEventListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addEventListener(listener)
       .build();
 
@@ -144,7 +154,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void send_event_when_process_requests_for_restart() {
     ManagedProcessEventListener listener = mock(ManagedProcessEventListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addEventListener(listener)
       .setWatcherDelayMs(1L)
       .build();
@@ -164,7 +174,7 @@ public class ManagedProcessHandlerTest {
 
   @Test
   public void stopForcibly_stops_the_process_without_graceful_request_for_stop() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
 
     try (TestManagedProcess testProcess = new TestManagedProcess()) {
       underTest.start(() -> testProcess);
@@ -183,7 +193,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void process_stops_after_graceful_request_for_stop() throws Exception {
     ProcessLifecycleListener listener = mock(ProcessLifecycleListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addProcessLifecycleListener(listener)
       .setHardStopTimeout(newTimeout(1, TimeUnit.HOURS))
       .build();
@@ -224,7 +234,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void process_is_stopped_forcibly_if_graceful_stop_is_too_long() throws Exception {
     ProcessLifecycleListener listener = mock(ProcessLifecycleListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addProcessLifecycleListener(listener)
       .setHardStopTimeout(newTimeout(1, TimeUnit.MILLISECONDS))
       .build();
@@ -246,7 +256,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void process_requests_are_listened_on_regular_basis() {
     ManagedProcessEventListener listener = mock(ManagedProcessEventListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addEventListener(listener)
       .setWatcherDelayMs(1L)
       .build();
@@ -262,7 +272,7 @@ public class ManagedProcessHandlerTest {
 
   @Test
   public void test_toString() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
     assertThat(underTest.toString()).isEqualTo("Process[" + A_PROCESS_ID.getKey() + "]");
   }
 
@@ -274,6 +284,7 @@ public class ManagedProcessHandlerTest {
     private boolean streamsClosed = false;
     private boolean operational = false;
     private boolean askedForRestart = false;
+    private boolean askedForStop = false;
     private boolean askedForHardStop = false;
     private boolean destroyedForcibly = false;
 
@@ -298,6 +309,12 @@ public class ManagedProcessHandlerTest {
     }
 
     @Override
+    public void askForStop() {
+      askedForStop = true;
+      // do not stop, just asking
+    }
+
+    @Override
     public void askForHardStop() {
       askedForHardStop = true;
       // do not stop, just asking
diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessLifecycleTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessLifecycleTest.java
index 3832d71c4ab..a2e6ff0fb9b 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessLifecycleTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessLifecycleTest.java
@@ -32,6 +32,7 @@ import static org.sonar.application.process.ManagedProcessLifecycle.State.INIT;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STARTED;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STARTING;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.HARD_STOPPING;
+import static org.sonar.application.process.ManagedProcessLifecycle.State.STOPPING;
 
 public class ManagedProcessLifecycleTest {
 
@@ -65,11 +66,11 @@ public class ManagedProcessLifecycleTest {
   }
 
   @Test
-  public void can_move_to_STOPPING_from_STARTING_STARTED_only() {
+  public void can_move_to_STOPPING_from_STARTING_STARTED_and_STOPPING_only() {
     for (ManagedProcessLifecycle.State state : ManagedProcessLifecycle.State.values()) {
       TestLifeCycleListener listener = new TestLifeCycleListener();
       boolean tryToMoveTo = newLifeCycle(state, listener).tryToMoveTo(HARD_STOPPING);
-      if (state == STARTING || state == STARTED) {
+      if (state == STARTING || state == STARTED || state == STOPPING) {
         assertThat(tryToMoveTo).as("from state " + state).isTrue();
         assertThat(listener.states).containsOnly(HARD_STOPPING);
       } else {