SONAR-12043 main process supports graceful and hard stop

24 files changed, 691 insertions, 130 deletions

diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/app/CeServer.java b/server/sonar-ce/src/main/java/org/sonar/ce/app/CeServer.java
index 9dfdd355547..1bc786c3909 100644
--- a/server/sonar-ce/src/main/java/org/sonar/ce/app/CeServer.java
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/app/CeServer.java
@@ -105,6 +105,12 @@ public class CeServer implements Monitored {
   }
 
   @Override
+  public void stop() {
+    // FIXME MMF-1673 implement graceful stop in CE: eg. call computeEngine.stopProcessing();
+    hardStop();
+  }
+
+  @Override
   public void hardStop() {
     if (ceMainThread != null) {
       // signal main Thread to stop
diff --git a/server/sonar-main/src/main/java/org/sonar/application/NodeLifecycle.java b/server/sonar-main/src/main/java/org/sonar/application/NodeLifecycle.java
index 4be33af9458..fe9830a8dd6 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/NodeLifecycle.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/NodeLifecycle.java
@@ -33,6 +33,7 @@ import static org.sonar.application.NodeLifecycle.State.OPERATIONAL;
 import static org.sonar.application.NodeLifecycle.State.RESTARTING;
 import static org.sonar.application.NodeLifecycle.State.STARTING;
 import static org.sonar.application.NodeLifecycle.State.STOPPED;
+import static org.sonar.application.NodeLifecycle.State.HARD_STOPPING;
 import static org.sonar.application.NodeLifecycle.State.STOPPING;
 
 /**
@@ -55,9 +56,12 @@ class NodeLifecycle {
     // at least one process is still stopping as part of a node restart
     RESTARTING,
 
-    // at least one process is still stopping
+    // at least one process is still stopping as part of a node graceful stop
     STOPPING,
 
+    // at least one process is still stopping as part of a node hard stop
+    HARD_STOPPING,
+
     // all processes are stopped
     STOPPED
   }
@@ -69,10 +73,11 @@ class NodeLifecycle {
   private static Map<State, Set<State>> buildTransitions() {
     Map<State, Set<State>> res = new EnumMap<>(State.class);
     res.put(INIT, toSet(STARTING));
-    res.put(STARTING, toSet(OPERATIONAL, RESTARTING, STOPPING, STOPPED));
-    res.put(OPERATIONAL, toSet(RESTARTING, STOPPING, STOPPED));
-    res.put(RESTARTING, toSet(STOPPING, STOPPED));
-    res.put(STOPPING, toSet(STOPPED));
+    res.put(STARTING, toSet(OPERATIONAL, RESTARTING, HARD_STOPPING, STOPPED));
+    res.put(OPERATIONAL, toSet(RESTARTING, STOPPING, HARD_STOPPING, STOPPED));
+    res.put(STOPPING, toSet(HARD_STOPPING, STOPPED));
+    res.put(RESTARTING, toSet(STARTING, HARD_STOPPING, STOPPED));
+    res.put(HARD_STOPPING, toSet(STOPPED));
     res.put(STOPPED, toSet(STARTING));
     return Collections.unmodifiableMap(res);
   }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/Scheduler.java b/server/sonar-main/src/main/java/org/sonar/application/Scheduler.java
index b01448b4787..39ca8b4d3e9 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/Scheduler.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/Scheduler.java
@@ -24,6 +24,11 @@ public interface Scheduler {
   void schedule() throws InterruptedException;
 
   /**
+   * Gracefully stops all processes and waits for them to be down.
+   */
+  void stop();
+
+  /**
    * Stops all processes and waits for them to be down.
    */
   void hardStop();
diff --git a/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java b/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java
index b218afc3ff1..e6e601d2b04 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
+import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.application.command.AbstractCommand;
@@ -38,11 +39,16 @@ import org.sonar.application.process.ManagedProcessLifecycle;
 import org.sonar.application.process.ProcessLifecycleListener;
 import org.sonar.process.ProcessId;
 
+import static org.sonar.application.NodeLifecycle.State.HARD_STOPPING;
+import static org.sonar.application.NodeLifecycle.State.RESTARTING;
+import static org.sonar.application.NodeLifecycle.State.STOPPED;
+import static org.sonar.application.NodeLifecycle.State.STOPPING;
 import static org.sonar.application.process.ManagedProcessHandler.Timeout.newTimeout;
 
 public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, ProcessLifecycleListener, AppStateListener {
 
   private static final Logger LOG = LoggerFactory.getLogger(SchedulerImpl.class);
+  private static final ManagedProcessHandler.Timeout HARD_STOP_TIMEOUT = newTimeout(1, TimeUnit.MINUTES);
 
   private final AppSettings settings;
   private final AppReloader appReloader;
@@ -56,6 +62,7 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
   private final EnumMap<ProcessId, ManagedProcessHandler> processesById = new EnumMap<>(ProcessId.class);
   private final AtomicInteger operationalCountDown = new AtomicInteger();
   private final AtomicInteger stopCountDown = new AtomicInteger(0);
+  private RestartStopperThread restartStopperThread;
   private HardStopperThread hardStopperThread;
   private RestarterThread restarterThread;
   private long processWatcherDelayMs = ManagedProcessHandler.DEFAULT_WATCHER_DELAY_MS;
@@ -87,8 +94,8 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
         .addProcessLifecycleListener(this)
         .addEventListener(this)
         .setWatcherDelayMs(processWatcherDelayMs)
-        // FIXME MMF-1673 timeout here must be changed to sonar.ce.task.timeout + 5 minutes if CE
-        .setHardStopTimeout(newTimeout(1, TimeUnit.MINUTES))
+        .setStopTimeout(stopTimeoutFor(processId))
+        .setHardStopTimeout(HARD_STOP_TIMEOUT)
         .build();
       processesById.put(process.getProcessId(), process);
     }
@@ -97,6 +104,21 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
     tryToStartAll();
   }
 
+  private static ManagedProcessHandler.Timeout stopTimeoutFor(ProcessId processId) {
+    switch (processId) {
+      case ELASTICSEARCH:
+        return HARD_STOP_TIMEOUT;
+      case WEB_SERVER:
+        return newTimeout(10, TimeUnit.MINUTES);
+      case COMPUTE_ENGINE:
+        // FIXME MMF-1673 make compute engine timeout configurable for ITs
+        return newTimeout(6, TimeUnit.HOURS);
+      case APP:
+      default:
+        throw new IllegalArgumentException("Unsupported processId " + processId);
+    }
+  }
+
   private void tryToStartAll() throws InterruptedException {
     tryToStartEs();
     tryToStartWeb();
@@ -155,9 +177,9 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
 
     try {
       processHandler.start(() -> {
-      AbstractCommand command = commandSupplier.get();
-      return processLauncher.launch(command);
-    });
+        AbstractCommand command = commandSupplier.get();
+        return processLauncher.launch(command);
+      });
     } catch (RuntimeException e) {
       // failed to start command -> stop everything
       hardStop();
@@ -165,23 +187,42 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
     }
   }
 
-  private void hardStopAll() throws InterruptedException {
+  @Override
+  public void stop() {
+    if (nodeLifecycle.tryToMoveTo(STOPPING)) {
+      LOG.info("Stopping SonarQube");
+    }
+    stopImpl();
+  }
+
+  private void stopImpl() {
+    try {
+      stopAll();
+      finalizeStop();
+    } catch (InterruptedException e) {
+      LOG.debug("Stop interrupted", e);
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  private void stopAll() throws InterruptedException {
     // order is important for non-cluster mode
-    hardStopProcess(ProcessId.COMPUTE_ENGINE);
-    hardStopProcess(ProcessId.WEB_SERVER);
-    hardStopProcess(ProcessId.ELASTICSEARCH);
+    stopProcess(ProcessId.COMPUTE_ENGINE);
+    stopProcess(ProcessId.WEB_SERVER);
+    stopProcess(ProcessId.ELASTICSEARCH);
   }
 
   /**
-   * Request for quick stop then blocks until process is stopped.
+   * Request for graceful stop then blocks until process is stopped.
    * Returns immediately if the process is disabled in configuration.
    *
    * @throws InterruptedException if {@link ManagedProcessHandler#hardStop()} throws a {@link InterruptedException}
    */
-  private void hardStopProcess(ProcessId processId) throws InterruptedException {
+  private void stopProcess(ProcessId processId) throws InterruptedException {
     ManagedProcessHandler process = processesById.get(processId);
     if (process != null) {
-      process.hardStop();
+      LOG.debug("Stopping [{}]...", process.getProcessId().getKey());
+      process.stop();
     }
   }
 
@@ -190,17 +231,16 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
    */
   @Override
   public void hardStop() {
-    if (nodeLifecycle.tryToMoveTo(NodeLifecycle.State.STOPPING)) {
-      LOG.info("Stopping SonarQube");
+    if (nodeLifecycle.tryToMoveTo(HARD_STOPPING)) {
+      LOG.info("Hard stopping SonarQube");
     }
+    hardStopImpl();
+  }
+
+  private void hardStopImpl() {
     try {
       hardStopAll();
-      if (hardStopperThread != null && Thread.currentThread() != hardStopperThread) {
-        hardStopperThread.interrupt();
-      }
-      if (restarterThread != null && Thread.currentThread() != restarterThread) {
-        restarterThread.interrupt();
-      }
+      finalizeStop();
     } catch (InterruptedException e) {
       // ignore and assume SQ stop is handled by another thread
       LOG.debug("Stopping all processes was interrupted in the middle of a hard stop" +
@@ -210,6 +250,49 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
     awaitTermination.countDown();
   }
 
+  private void hardStopAll() throws InterruptedException {
+    // order is important for non-cluster mode
+    hardStopProcess(ProcessId.COMPUTE_ENGINE);
+    hardStopProcess(ProcessId.WEB_SERVER);
+    hardStopProcess(ProcessId.ELASTICSEARCH);
+
+    // if all process are already stopped (may occur, eg., when stopping because restart of 1st process failed),
+    // node state won't be updated on process stopped callback, so we must ensure
+    // the node's state is updated
+    if (nodeLifecycle.getState() != RESTARTING) {
+      nodeLifecycle.tryToMoveTo(STOPPED);
+    }
+  }
+
+  private void finalizeStop() {
+    if (nodeLifecycle.getState() == STOPPED) {
+      interrupt(restartStopperThread);
+      interrupt(hardStopperThread);
+      interrupt(restarterThread);
+    }
+  }
+
+  private static void interrupt(@Nullable Thread thread) {
+    if (thread != null
+      // do not interrupt oneself
+      && Thread.currentThread() != thread) {
+      thread.interrupt();
+    }
+  }
+
+  /**
+   * Request for graceful stop then blocks until process is stopped.
+   * Returns immediately if the process is disabled in configuration.
+   *
+   * @throws InterruptedException if {@link ManagedProcessHandler#hardStop()} throws a {@link InterruptedException}
+   */
+  private void hardStopProcess(ProcessId processId) throws InterruptedException {
+    ManagedProcessHandler process = processesById.get(processId);
+    if (process != null) {
+      process.hardStop();
+    }
+  }
+
   @Override
   public void awaitTermination() {
     try {
@@ -223,16 +306,17 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
   public void onManagedProcessEvent(ProcessId processId, Type type) {
     if (type == Type.OPERATIONAL) {
       onProcessOperational(processId);
-    } else if (type == Type.ASK_FOR_RESTART && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.RESTARTING)) {
+    } else if (type == Type.ASK_FOR_RESTART && nodeLifecycle.tryToMoveTo(RESTARTING)) {
       LOG.info("SQ restart requested by Process[{}]", processId.getKey());
-      hardStopAsync();
+      stopAsyncForRestart();
     }
   }
 
   private void onProcessOperational(ProcessId processId) {
     LOG.info("Process[{}] is up", processId.getKey());
     appState.setOperational(processId);
-    if (operationalCountDown.decrementAndGet() == 0 && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.OPERATIONAL)) {
+    boolean lastProcessStarted = operationalCountDown.decrementAndGet() == 0;
+    if (lastProcessStarted && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.OPERATIONAL)) {
       LOG.info("SonarQube is up");
     }
   }
@@ -271,13 +355,14 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
     boolean lastProcessStopped = stopCountDown.decrementAndGet() == 0;
     switch (nodeLifecycle.getState()) {
       case RESTARTING:
-        if (lastProcessStopped && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.STOPPED)) {
+        if (lastProcessStopped) {
           LOG.info("SonarQube is restarting");
           restartAsync();
         }
         break;
-      case STOPPING:
-        if (lastProcessStopped && nodeLifecycle.tryToMoveTo(NodeLifecycle.State.STOPPED)) {
+      case HARD_STOPPING:
+        if (lastProcessStopped && nodeLifecycle.tryToMoveTo(STOPPED)) {
+          LOG.info("SonarQube is stopped");
           // all processes are stopped, no restart requested
           // Let's clean-up resources
           hardStop();
@@ -290,11 +375,31 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
   }
 
   private void hardStopAsync() {
+    if (hardStopperThread != null) {
+      LOG.debug("Hard stopper thread was not null (name is \"{}\")", hardStopperThread.getName(), new Exception());
+      hardStopperThread.interrupt();
+    }
+
     hardStopperThread = new HardStopperThread();
     hardStopperThread.start();
   }
 
+  private void stopAsyncForRestart() {
+    if (restartStopperThread != null) {
+      LOG.debug("Restart stopper thread was not null", new Exception());
+      restartStopperThread.interrupt();
+    }
+
+    restartStopperThread = new RestartStopperThread();
+    restartStopperThread.start();
+  }
+
   private void restartAsync() {
+    if (restarterThread != null) {
+      LOG.debug("Restarter thread was not null (name is \"{}\")", restarterThread.getName(), new Exception());
+      restarterThread.interrupt();
+    }
+
     restarterThread = new RestarterThread();
     restarterThread.start();
   }
@@ -314,24 +419,33 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
         LOG.debug("{} thread was interrupted", getName(), e);
         super.interrupt();
       } catch (Exception e) {
-        LOG.error("Fail to restart", e);
+        LOG.error("Failed to restart", e);
         hardStop();
       }
     }
   }
 
+  private class RestartStopperThread extends Thread {
+    public RestartStopperThread() {
+      super("Restart stopper");
+    }
+
+    @Override
+    public void run() {
+      stopImpl();
+    }
+  }
+
   private class HardStopperThread extends Thread {
+
     public HardStopperThread() {
       super("Hard stopper");
     }
 
     @Override
     public void run() {
-      try {
-        hardStopAll();
-      } catch (InterruptedException e) {
-        LOG.debug("{} thread was interrupted", getName(), e);
-        interrupt();
+      if (nodeLifecycle.tryToMoveTo(HARD_STOPPING)) {
+        hardStopImpl();
       }
     }
   }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
index c8e0badbe77..fd028f5cd10 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
@@ -115,6 +115,11 @@ public class EsManagedProcess extends AbstractManagedProcess {
   }
 
   @Override
+  public void askForStop() {
+    askForHardStop();
+  }
+
+  @Override
   public void askForHardStop() {
     process.destroy();
   }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcess.java
index 9cb0028e0a1..14eccb184b6 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcess.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcess.java
@@ -67,6 +67,8 @@ public interface ManagedProcess {
    */
   boolean isOperational();
 
+  void askForStop();
+
   /**
    * Send request to quick stop to the process
    */
@@ -84,5 +86,4 @@ public interface ManagedProcess {
    * Child process will typically stop sending the signal requesting restart from now on.
    */
   void acknowledgeAskForRestart();
-
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessHandler.java b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessHandler.java
index 791bc351ae4..1104833d6fd 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessHandler.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessHandler.java
@@ -40,6 +40,7 @@ public class ManagedProcessHandler {
   private final ProcessId processId;
   private final ManagedProcessLifecycle lifecycle;
   private final List<ManagedProcessEventListener> eventListeners;
+  private final Timeout stopTimeout;
   private final Timeout hardStopTimeout;
   private final long watcherDelayMs;
 
@@ -56,6 +57,7 @@ public class ManagedProcessHandler {
     this.processId = requireNonNull(builder.processId, "processId can't be null");
     this.lifecycle = new ManagedProcessLifecycle(this.processId, builder.lifecycleListeners);
     this.eventListeners = builder.eventListeners;
+    this.stopTimeout = builder.stopTimeout;
     this.hardStopTimeout = builder.hardStopTimeout;
     this.watcherDelayMs = builder.watcherDelayMs;
     this.stopWatcher = new StopWatcher();
@@ -94,6 +96,20 @@ public class ManagedProcessHandler {
     return lifecycle.getState();
   }
 
+  public void stop() throws InterruptedException {
+    if (lifecycle.tryToMoveTo(ManagedProcessLifecycle.State.STOPPING)) {
+      stopImpl();
+      if (process != null && process.isAlive()) {
+        LOG.info("{} failed to stop in a graceful fashion. Hard stopping it.", processId.getKey());
+      }
+      // enforce stop and clean-up even if process has been quickly stopped
+      stopForcibly();
+    } else {
+      // already stopping or stopped
+      waitForDown();
+    }
+  }
+
   /**
    * Sends kill signal and awaits termination. No guarantee that process is gracefully terminated (=shutdown hooks
    * executed). It depends on OS.
@@ -123,6 +139,21 @@ public class ManagedProcessHandler {
     }
   }
 
+  private void stopImpl() throws InterruptedException {
+    if (process == null) {
+      return;
+    }
+    try {
+      process.askForStop();
+      process.waitFor(stopTimeout.getDuration(), stopTimeout.getUnit());
+    } catch (InterruptedException e) {
+      // can't wait for the termination of process. Let's assume it's down.
+      throw rethrowWithWarn(e, format("Interrupted while stopping process %s", processId));
+    } catch (Throwable e) {
+      LOG.error("Failed asking for graceful stop of process {}", processId, e);
+    }
+  }
+
   private void hardStopImpl() throws InterruptedException {
     if (process == null) {
       return;
@@ -132,15 +163,18 @@ public class ManagedProcessHandler {
       process.waitFor(hardStopTimeout.getDuration(), hardStopTimeout.getUnit());
     } catch (InterruptedException e) {
       // can't wait for the termination of process. Let's assume it's down.
-      String errorMessage = format("Interrupted while hard stopping process %s", processId);
-      LOG.warn(errorMessage, e);
-      Thread.currentThread().interrupt();
-      throw new InterruptedException(errorMessage);
+      throw rethrowWithWarn(e, format("Interrupted while hard stopping process %s", processId));
     } catch (Throwable e) {
       LOG.error("Failed while asking for hard stop of process {}", processId, e);
     }
   }
 
+  private static InterruptedException rethrowWithWarn(InterruptedException e, String errorMessage) {
+    LOG.warn(errorMessage, e);
+    Thread.currentThread().interrupt();
+    return new InterruptedException(errorMessage);
+  }
+
   public void stopForcibly() {
     eventWatcher.interrupt();
     stopWatcher.interrupt();
@@ -242,7 +276,8 @@ public class ManagedProcessHandler {
     private final List<ManagedProcessEventListener> eventListeners = new ArrayList<>();
     private final List<ProcessLifecycleListener> lifecycleListeners = new ArrayList<>();
     private long watcherDelayMs = DEFAULT_WATCHER_DELAY_MS;
-    private Timeout hardStopTimeout = new Timeout(1, TimeUnit.MINUTES);
+    private Timeout stopTimeout;
+    private Timeout hardStopTimeout;
 
     private Builder(ProcessId processId) {
       this.processId = processId;
@@ -266,12 +301,27 @@ public class ManagedProcessHandler {
       return this;
     }
 
+    public Builder setStopTimeout(Timeout stopTimeout) {
+      this.stopTimeout = ensureStopTimeoutNonNull(stopTimeout);
+      return this;
+    }
+
     public Builder setHardStopTimeout(Timeout hardStopTimeout) {
-      this.hardStopTimeout = requireNonNull(hardStopTimeout, "hardStopTimeout can't be null");
+      this.hardStopTimeout = ensureHardStopTimeoutNonNull(hardStopTimeout);
       return this;
     }
 
+    private static Timeout ensureStopTimeoutNonNull(Timeout stopTimeout) {
+      return requireNonNull(stopTimeout, "stopTimeout can't be null");
+    }
+
+    private static Timeout ensureHardStopTimeoutNonNull(Timeout hardStopTimeout) {
+      return requireNonNull(hardStopTimeout, "hardStopTimeout can't be null");
+    }
+
     public ManagedProcessHandler build() {
+      ensureStopTimeoutNonNull(this.stopTimeout);
+      ensureHardStopTimeoutNonNull(this.hardStopTimeout);
       return new ManagedProcessHandler(this);
     }
   }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessLifecycle.java b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessLifecycle.java
index dc103b4f8db..9c4ea29a9a0 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessLifecycle.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessLifecycle.java
@@ -30,16 +30,17 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.process.ProcessId;
 
+import static org.sonar.application.process.ManagedProcessLifecycle.State.HARD_STOPPING;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.INIT;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STARTED;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STARTING;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STOPPED;
-import static org.sonar.application.process.ManagedProcessLifecycle.State.HARD_STOPPING;
+import static org.sonar.application.process.ManagedProcessLifecycle.State.STOPPING;
 
 public class ManagedProcessLifecycle {
 
   public enum State {
-    INIT, STARTING, STARTED, HARD_STOPPING, STOPPED
+    INIT, STARTING, STARTED, STOPPING, HARD_STOPPING, STOPPED
   }
 
   private static final Logger LOG = LoggerFactory.getLogger(ManagedProcessLifecycle.class);
@@ -62,8 +63,9 @@ public class ManagedProcessLifecycle {
   private static Map<State, Set<State>> buildTransitions() {
     Map<State, Set<State>> res = new EnumMap<>(State.class);
     res.put(INIT, toSet(STARTING));
-    res.put(STARTING, toSet(STARTED, HARD_STOPPING, STOPPED));
-    res.put(STARTED, toSet(HARD_STOPPING, STOPPED));
+    res.put(STARTING, toSet(STARTED, STOPPING, HARD_STOPPING, STOPPED));
+    res.put(STARTED, toSet(STOPPING, HARD_STOPPING, STOPPED));
+    res.put(STOPPING, toSet(HARD_STOPPING, STOPPED));
     res.put(HARD_STOPPING, toSet(STOPPED));
     res.put(STOPPED, toSet());
     return Collections.unmodifiableMap(res);
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessCommandsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessCommandsManagedProcess.java
index 975d746193c..4ce128bd56f 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessCommandsManagedProcess.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessCommandsManagedProcess.java
@@ -42,6 +42,14 @@ public class ProcessCommandsManagedProcess extends AbstractManagedProcess {
   }
 
   /**
+   * Send request to gracefully stop to the process (via ipc shared memory)
+   */
+  @Override
+  public void askForStop() {
+    commands.askForStop();
+  }
+
+  /**
    * Send request to quickly stop to the process (via ipc shared memory)
    */
   @Override
diff --git a/server/sonar-main/src/test/java/org/sonar/application/NodeLifecycleTest.java b/server/sonar-main/src/test/java/org/sonar/application/NodeLifecycleTest.java
new file mode 100644
index 00000000000..5ebfd9587fd
--- /dev/null
+++ b/server/sonar-main/src/test/java/org/sonar/application/NodeLifecycleTest.java
@@ -0,0 +1,106 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.sonar.application.NodeLifecycle.State.HARD_STOPPING;
+import static org.sonar.application.NodeLifecycle.State.INIT;
+import static org.sonar.application.NodeLifecycle.State.OPERATIONAL;
+import static org.sonar.application.NodeLifecycle.State.RESTARTING;
+import static org.sonar.application.NodeLifecycle.State.STARTING;
+import static org.sonar.application.NodeLifecycle.State.STOPPED;
+import static org.sonar.application.NodeLifecycle.State.STOPPING;
+
+public class NodeLifecycleTest {
+  private NodeLifecycle underTest = new NodeLifecycle();
+
+  @Test
+  public void verify_regular_start_and_graceful_stop_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPING);
+    assertThat(underTest.tryToMoveTo(STOPPED)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPED);
+  }
+
+  @Test
+  public void verify_start_and_hard_stop_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(HARD_STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(HARD_STOPPING);
+    assertThat(underTest.tryToMoveTo(STOPPED)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPED);
+  }
+
+  @Test
+  public void verify_start_graceful_stop_interrupted_by_hard_stop_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPING);
+    assertThat(underTest.tryToMoveTo(HARD_STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(HARD_STOPPING);
+    assertThat(underTest.tryToMoveTo(STOPPED)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPED);
+  }
+
+  @Test
+  public void verify_regular_start_restart_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(RESTARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(RESTARTING);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+  }
+
+  @Test
+  public void verify_failed_restart_resulting_in_hard_stop_cycle() {
+    assertThat(underTest.getState()).isEqualTo(INIT);
+    assertThat(underTest.tryToMoveTo(STARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STARTING);
+    assertThat(underTest.tryToMoveTo(OPERATIONAL)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(OPERATIONAL);
+    assertThat(underTest.tryToMoveTo(RESTARTING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(RESTARTING);
+    assertThat(underTest.tryToMoveTo(HARD_STOPPING)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(HARD_STOPPING);
+    assertThat(underTest.tryToMoveTo(STOPPED)).isTrue();
+    assertThat(underTest.getState()).isEqualTo(STOPPED);
+  }
+}
diff --git a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java
index 36e5030c6f7..533e9abc932 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java
@@ -127,6 +127,44 @@ public class SchedulerImplTest {
     processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isTrue());
 
     // processes are stopped in reverse order of startup
+    underTest.stop();
+    assertThat(orderedStops).containsExactly(COMPUTE_ENGINE, WEB_SERVER, ELASTICSEARCH);
+    processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse());
+
+    // does nothing because scheduler is already terminated
+    underTest.awaitTermination();
+  }
+
+  @Test
+  public void start_and_hard_stop_sequence_of_ES_WEB_CE_in_order() throws Exception {
+    SchedulerImpl underTest = newScheduler(false);
+    underTest.schedule();
+
+    // elasticsearch does not have preconditions to start
+    TestManagedProcess es = processLauncher.waitForProcess(ELASTICSEARCH);
+    assertThat(es.isAlive()).isTrue();
+    assertThat(processLauncher.processes).hasSize(1);
+
+    // elasticsearch becomes operational -> web leader is starting
+    es.operational = true;
+    waitForAppStateOperational(appState, ELASTICSEARCH);
+    TestManagedProcess web = processLauncher.waitForProcess(WEB_SERVER);
+    assertThat(web.isAlive()).isTrue();
+    assertThat(processLauncher.processes).hasSize(2);
+    assertThat(processLauncher.commands).containsExactly(esScriptCommand, webLeaderCommand);
+
+    // web becomes operational -> CE is starting
+    web.operational = true;
+    waitForAppStateOperational(appState, WEB_SERVER);
+    TestManagedProcess ce = processLauncher.waitForProcess(COMPUTE_ENGINE);
+    assertThat(ce.isAlive()).isTrue();
+    assertThat(processLauncher.processes).hasSize(3);
+    assertThat(processLauncher.commands).containsExactly(esScriptCommand, webLeaderCommand, ceCommand);
+
+    // all processes are up
+    processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isTrue());
+
+    // processes are stopped in reverse order of startup
     underTest.hardStop();
     assertThat(orderedStops).containsExactly(COMPUTE_ENGINE, WEB_SERVER, ELASTICSEARCH);
     processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse());
@@ -353,7 +391,7 @@ public class SchedulerImplTest {
     private ManagedProcess launchImpl(AbstractCommand<?> javaCommand) {
       commands.add(javaCommand);
       if (makeStartupFail == javaCommand.getProcessId()) {
-        throw new IllegalStateException("cannot start " + javaCommand.getProcessId());
+        throw new IllegalStateException("Faking startup of java command failing for " + javaCommand.getProcessId());
       }
       TestManagedProcess process = new TestManagedProcess(javaCommand.getProcessId());
       processes.put(javaCommand.getProcessId(), process);
@@ -402,6 +440,7 @@ public class SchedulerImplTest {
     private final ProcessId processId;
     private final CountDownLatch alive = new CountDownLatch(1);
     private boolean operational = false;
+    private boolean askedForStop = false;
     private boolean askedForRestart = false;
 
     private TestManagedProcess(ProcessId processId) {
@@ -428,6 +467,12 @@ public class SchedulerImplTest {
     }
 
     @Override
+    public void askForStop() {
+      this.askedForStop = true;
+      destroyForcibly();
+    }
+
+    @Override
     public void askForHardStop() {
       destroyForcibly();
     }
diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessHandlerTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessHandlerTest.java
index 882ac36a1f9..ae395897423 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessHandlerTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessHandlerTest.java
@@ -49,7 +49,7 @@ public class ManagedProcessHandlerTest {
 
   @Test
   public void initial_state_is_INIT() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
 
     assertThat(underTest.getProcessId()).isEqualTo(A_PROCESS_ID);
     assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.INIT);
@@ -58,7 +58,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void start_and_stop_process() {
     ProcessLifecycleListener listener = mock(ProcessLifecycleListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addProcessLifecycleListener(listener)
       .build();
 
@@ -79,34 +79,44 @@ public class ManagedProcessHandlerTest {
     }
   }
 
+  public ManagedProcessHandler.Builder newHanderBuilder(ProcessId aProcessId) {
+    return ManagedProcessHandler.builder(aProcessId)
+      .setStopTimeout(newTimeout(1, TimeUnit.SECONDS))
+      .setHardStopTimeout(newTimeout(1, TimeUnit.SECONDS));
+  }
+
   @Test
   public void start_does_not_nothing_if_already_started_once() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
 
     try (TestManagedProcess testProcess = new TestManagedProcess()) {
       assertThat(underTest.start(() -> testProcess)).isTrue();
       assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.STARTED);
 
-      assertThat(underTest.start(() -> {throw new IllegalStateException();})).isFalse();
+      assertThat(underTest.start(() -> {
+        throw new IllegalStateException();
+      })).isFalse();
       assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.STARTED);
     }
   }
 
   @Test
   public void start_throws_exception_and_move_to_state_STOPPED_if_execution_of_command_fails() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
 
     expectedException.expect(IllegalStateException.class);
     expectedException.expectMessage("error");
 
-    underTest.start(() -> {throw new IllegalStateException("error");});
+    underTest.start(() -> {
+      throw new IllegalStateException("error");
+    });
     assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.STOPPED);
   }
 
   @Test
   public void send_event_when_process_is_operational() {
     ManagedProcessEventListener listener = mock(ManagedProcessEventListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addEventListener(listener)
       .build();
 
@@ -124,7 +134,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void operational_event_is_sent_once() {
     ManagedProcessEventListener listener = mock(ManagedProcessEventListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addEventListener(listener)
       .build();
 
@@ -144,7 +154,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void send_event_when_process_requests_for_restart() {
     ManagedProcessEventListener listener = mock(ManagedProcessEventListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addEventListener(listener)
       .setWatcherDelayMs(1L)
       .build();
@@ -164,7 +174,7 @@ public class ManagedProcessHandlerTest {
 
   @Test
   public void stopForcibly_stops_the_process_without_graceful_request_for_stop() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
 
     try (TestManagedProcess testProcess = new TestManagedProcess()) {
       underTest.start(() -> testProcess);
@@ -183,7 +193,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void process_stops_after_graceful_request_for_stop() throws Exception {
     ProcessLifecycleListener listener = mock(ProcessLifecycleListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addProcessLifecycleListener(listener)
       .setHardStopTimeout(newTimeout(1, TimeUnit.HOURS))
       .build();
@@ -224,7 +234,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void process_is_stopped_forcibly_if_graceful_stop_is_too_long() throws Exception {
     ProcessLifecycleListener listener = mock(ProcessLifecycleListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addProcessLifecycleListener(listener)
       .setHardStopTimeout(newTimeout(1, TimeUnit.MILLISECONDS))
       .build();
@@ -246,7 +256,7 @@ public class ManagedProcessHandlerTest {
   @Test
   public void process_requests_are_listened_on_regular_basis() {
     ManagedProcessEventListener listener = mock(ManagedProcessEventListener.class);
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID)
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addEventListener(listener)
       .setWatcherDelayMs(1L)
       .build();
@@ -262,7 +272,7 @@ public class ManagedProcessHandlerTest {
 
   @Test
   public void test_toString() {
-    ManagedProcessHandler underTest = ManagedProcessHandler.builder(A_PROCESS_ID).build();
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID).build();
     assertThat(underTest.toString()).isEqualTo("Process[" + A_PROCESS_ID.getKey() + "]");
   }
 
@@ -274,6 +284,7 @@ public class ManagedProcessHandlerTest {
     private boolean streamsClosed = false;
     private boolean operational = false;
     private boolean askedForRestart = false;
+    private boolean askedForStop = false;
     private boolean askedForHardStop = false;
     private boolean destroyedForcibly = false;
 
@@ -298,6 +309,12 @@ public class ManagedProcessHandlerTest {
     }
 
     @Override
+    public void askForStop() {
+      askedForStop = true;
+      // do not stop, just asking
+    }
+
+    @Override
     public void askForHardStop() {
       askedForHardStop = true;
       // do not stop, just asking
diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessLifecycleTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessLifecycleTest.java
index 3832d71c4ab..a2e6ff0fb9b 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessLifecycleTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessLifecycleTest.java
@@ -32,6 +32,7 @@ import static org.sonar.application.process.ManagedProcessLifecycle.State.INIT;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STARTED;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.STARTING;
 import static org.sonar.application.process.ManagedProcessLifecycle.State.HARD_STOPPING;
+import static org.sonar.application.process.ManagedProcessLifecycle.State.STOPPING;
 
 public class ManagedProcessLifecycleTest {
 
@@ -65,11 +66,11 @@ public class ManagedProcessLifecycleTest {
   }
 
   @Test
-  public void can_move_to_STOPPING_from_STARTING_STARTED_only() {
+  public void can_move_to_STOPPING_from_STARTING_STARTED_and_STOPPING_only() {
     for (ManagedProcessLifecycle.State state : ManagedProcessLifecycle.State.values()) {
       TestLifeCycleListener listener = new TestLifeCycleListener();
       boolean tryToMoveTo = newLifeCycle(state, listener).tryToMoveTo(HARD_STOPPING);
-      if (state == STARTING || state == STARTED) {
+      if (state == STARTING || state == STARTED || state == STOPPING) {
         assertThat(tryToMoveTo).as("from state " + state).isTrue();
         assertThat(listener.states).containsOnly(HARD_STOPPING);
       } else {
diff --git a/server/sonar-process/build.gradle b/server/sonar-process/build.gradle
index d458ec793f0..ee525939cb9 100644
--- a/server/sonar-process/build.gradle
+++ b/server/sonar-process/build.gradle
@@ -27,5 +27,6 @@ dependencies {
   testCompile 'org.assertj:assertj-core'
   testCompile 'org.eclipse.jetty:jetty-server'
   testCompile 'org.mockito:mockito-core'
+  testCompile 'org.awaitility:awaitility'
   testCompile project(':sonar-testing-harness')
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/HardStopWatcher.java b/server/sonar-process/src/main/java/org/sonar/process/AbstractStopWatcher.java
index 7286d64259a..50e4e73940a 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/HardStopWatcher.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/AbstractStopWatcher.java
@@ -19,36 +19,32 @@
  */
 package org.sonar.process;
 
-import org.slf4j.LoggerFactory;
-import org.sonar.process.sharedmemoryfile.ProcessCommands;
+import com.google.common.annotations.VisibleForTesting;
+import java.util.function.BooleanSupplier;
 
-/**
- * This watchdog is looking for hard stop to be requested via {@link ProcessCommands#askedForHardStop()}.
- */
-public class HardStopWatcher extends Thread {
-
-  private final ProcessCommands commands;
-  private final Stoppable stoppable;
+abstract class AbstractStopWatcher extends Thread {
+  private final Runnable stopCommand;
+  private final BooleanSupplier shouldStopTest;
   private final long delayMs;
-  private boolean watching = true;
+  private volatile boolean watching = true;
 
-  public HardStopWatcher(ProcessCommands commands, Stoppable stoppable) {
-    this(commands, stoppable, 500L);
+  public AbstractStopWatcher(String threadName, Runnable stopCommand, BooleanSupplier shouldStopTest) {
+    this(threadName, stopCommand, shouldStopTest, 500L);
   }
 
-  HardStopWatcher(ProcessCommands commands, Stoppable stoppable, long delayMs) {
-    super("HardStop Watcher");
-    this.commands = commands;
-    this.stoppable = stoppable;
+  @VisibleForTesting
+  AbstractStopWatcher(String threadName, Runnable stopCommand, BooleanSupplier shouldStopTest, long delayMs) {
+    super(threadName);
+    this.stopCommand = stopCommand;
+    this.shouldStopTest = shouldStopTest;
     this.delayMs = delayMs;
   }
 
   @Override
   public void run() {
     while (watching) {
-      if (commands.askedForHardStop()) {
-        LoggerFactory.getLogger(getClass()).info("Hard stopping process");
-        stoppable.hardStopAsync();
+      if (shouldStopTest.getAsBoolean()) {
+        stopCommand.run();
         watching = false;
       } else {
         try {
@@ -63,6 +59,7 @@ public class HardStopWatcher extends Thread {
   }
 
   public void stopWatching() {
+    super.interrupt();
     watching = false;
   }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/HardStopperThread.java b/server/sonar-process/src/main/java/org/sonar/process/AbstractStopperThread.java
index c2414dd17ba..ae0c85d0018 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/HardStopperThread.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/AbstractStopperThread.java
@@ -28,14 +28,15 @@ import org.slf4j.LoggerFactory;
 /**
  * Stops process in a short time fashion
  */
-class HardStopperThread extends Thread {
+abstract class AbstractStopperThread extends Thread {
 
-  private final Monitored monitored;
+  private final Runnable stopCode;
   private final long terminationTimeoutMs;
+  private boolean stop = false;
 
-  HardStopperThread(Monitored monitored, long terminationTimeoutMs) {
-    super("HardStopper");
-    this.monitored = monitored;
+  AbstractStopperThread(String threadName, Runnable stopCode, long terminationTimeoutMs) {
+    super(threadName);
+    this.stopCode = stopCode;
     this.terminationTimeoutMs = terminationTimeoutMs;
   }
 
@@ -43,11 +44,18 @@ class HardStopperThread extends Thread {
   public void run() {
     ExecutorService executor = Executors.newSingleThreadExecutor();
     try {
-      Future future = executor.submit(monitored::hardStop);
+      Future future = executor.submit(stopCode);
       future.get(terminationTimeoutMs, TimeUnit.MILLISECONDS);
     } catch (Exception e) {
-      LoggerFactory.getLogger(getClass()).error("Can not stop in {}ms", terminationTimeoutMs, e);
+      if (!stop) {
+        LoggerFactory.getLogger(getClass()).error("Can not stop in {}ms", terminationTimeoutMs, e);
+      }
     }
     executor.shutdownNow();
   }
+
+  public void stopIt() {
+    this.stop = true;
+    super.interrupt();
+  }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/Monitored.java b/server/sonar-process/src/main/java/org/sonar/process/Monitored.java
index 56d6b249ce0..653f5408220 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/Monitored.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/Monitored.java
@@ -47,6 +47,11 @@ public interface Monitored {
   void awaitStop();
 
   /**
+   * Ask process to gracefully stop and wait until then.
+   */
+  void stop();
+
+  /**
    * Ask process to quickly stop and wait until then.
    */
   void hardStop();
diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java
index ae6b4376557..1976f5699b4 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java
@@ -20,12 +20,17 @@
 package org.sonar.process;
 
 import java.io.File;
+import java.util.Optional;
+import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.process.sharedmemoryfile.DefaultProcessCommands;
 import org.sonar.process.sharedmemoryfile.ProcessCommands;
 
-public class ProcessEntryPoint implements Stoppable {
+import static java.util.Optional.of;
+import static java.util.Optional.ofNullable;
+
+public class ProcessEntryPoint {
 
   public static final String PROPERTY_PROCESS_KEY = "process.key";
   public static final String PROPERTY_PROCESS_INDEX = "process.index";
@@ -39,18 +44,19 @@ public class ProcessEntryPoint implements Stoppable {
   private final Lifecycle lifecycle = new Lifecycle();
   private final ProcessCommands commands;
   private final SystemExit exit;
+  private final StopWatcher stopWatcher;
   private final HardStopWatcher hardStopWatcher;
-  private volatile Monitored monitored;
-  private volatile HardStopperThread hardStopperThread;
-
   // new Runnable() is important to avoid conflict of call to ProcessEntryPoint#stop() with Thread#stop()
-  private Thread shutdownHook = new Thread(new Runnable() {
+  private final Thread shutdownHook = new Thread(new Runnable() {
     @Override
     public void run() {
       exit.setInShutdownHook();
-      hardStop();
+      stop();
     }
   });
+  private volatile Monitored monitored;
+  private volatile StopperThread stopperThread;
+  private volatile HardStopperThread hardStopperThread;
 
   ProcessEntryPoint(Props props, SystemExit exit, ProcessCommands commands) {
     this(props, getProcessNumber(props), getSharedDir(props), exit, commands);
@@ -63,6 +69,7 @@ public class ProcessEntryPoint implements Stoppable {
     this.sharedDir = sharedDir;
     this.exit = exit;
     this.commands = commands;
+    this.stopWatcher = new StopWatcher(commands, this);
     this.hardStopWatcher = new HardStopWatcher(commands, this);
   }
 
@@ -108,6 +115,7 @@ public class ProcessEntryPoint implements Stoppable {
   private void launch(Logger logger) throws InterruptedException {
     logger.info("Starting {}", getKey());
     Runtime.getRuntime().addShutdownHook(shutdownHook);
+    stopWatcher.start();
     hardStopWatcher.start();
 
     monitored.start();
@@ -152,31 +160,63 @@ public class ProcessEntryPoint implements Stoppable {
     return lifecycle.getState() == Lifecycle.State.STARTED;
   }
 
+  void stop() {
+    stopAsync()
+      .ifPresent(stoppingThread -> {
+        try {
+          // join() does nothing if thread already finished
+          stoppingThread.join();
+          lifecycle.tryToMoveTo(Lifecycle.State.STOPPED);
+          commands.endWatch();
+          exit.exit(0);
+        } catch (InterruptedException e) {
+          // stop can be aborted by a hard stop
+          Thread.currentThread().interrupt();
+        }
+      });
+  }
+
+  private Optional<StopperThread> stopAsync() {
+    if (lifecycle.tryToMoveTo(Lifecycle.State.STOPPING)) {
+      long terminationTimeoutMs = Long.parseLong(props.nonNullValue(PROPERTY_TERMINATION_TIMEOUT_MS));
+      stopperThread = new StopperThread(monitored, terminationTimeoutMs);
+      stopperThread.start();
+      stopWatcher.stopWatching();
+      return of(stopperThread);
+    }
+    // stopperThread could already exist
+    return ofNullable(stopperThread);
+  }
+
   /**
    * Blocks until stopped in a timely fashion (see {@link HardStopperThread})
    */
   void hardStop() {
-    hardStopAsync();
-    try {
-      // hardStopperThread is not null for sure
-      // join() does nothing if thread already finished
-      hardStopperThread.join();
-      lifecycle.tryToMoveTo(Lifecycle.State.STOPPED);
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-    }
-    commands.endWatch();
-    exit.exit(0);
+    hardStopAsync()
+      .ifPresent(stoppingThread -> {
+        try {
+          // join() does nothing if thread already finished
+          stoppingThread.join();
+          lifecycle.tryToMoveTo(Lifecycle.State.STOPPED);
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+        }
+        commands.endWatch();
+        exit.exit(0);
+      });
   }
 
-  @Override
-  public void hardStopAsync() {
+  private Optional<HardStopperThread> hardStopAsync() {
     if (lifecycle.tryToMoveTo(Lifecycle.State.HARD_STOPPING)) {
       long terminationTimeoutMs = Long.parseLong(props.nonNullValue(PROPERTY_TERMINATION_TIMEOUT_MS));
-      hardStopperThread = new HardStopperThread(monitored, terminationTimeoutMs);
+      hardStopperThread = new HardStopperThread(monitored, terminationTimeoutMs, stopperThread);
       hardStopperThread.start();
       hardStopWatcher.stopWatching();
+      stopWatcher.stopWatching();
+      return of(hardStopperThread);
     }
+    // hardStopperThread could already exist
+    return ofNullable(hardStopperThread);
   }
 
   Lifecycle.State getState() {
@@ -202,4 +242,68 @@ public class ProcessEntryPoint implements Stoppable {
   private static File getSharedDir(Props props) {
     return props.nonNullValueAsFile(PROPERTY_SHARED_PATH);
   }
+
+  /**
+   * This watchdog is looking for hard stop to be requested via {@link ProcessCommands#askedForHardStop()}.
+   */
+  private static class HardStopWatcher extends AbstractStopWatcher {
+
+    private HardStopWatcher(ProcessCommands commands, ProcessEntryPoint processEntryPoint) {
+      super(
+        "HardStop Watcher",
+        () -> {
+          LoggerFactory.getLogger(HardStopWatcher.class).info("Hard stopping process");
+          processEntryPoint.hardStopAsync();
+        },
+        commands::askedForHardStop);
+    }
+
+  }
+
+  /**
+   * This watchdog is looking for graceful stop to be requested via {@link ProcessCommands#askedForStop()} ()}.
+   */
+  private static class StopWatcher extends AbstractStopWatcher {
+
+    private StopWatcher(ProcessCommands commands, ProcessEntryPoint processEntryPoint) {
+      super(
+        "Stop Watcher",
+        () -> {
+          LoggerFactory.getLogger(StopWatcher.class).info("Stopping process");
+          processEntryPoint.stopAsync();
+        },
+        commands::askedForStop);
+    }
+
+  }
+
+  /**
+   * Stops process in a graceful fashion
+   */
+  private static class StopperThread extends AbstractStopperThread {
+
+    private StopperThread(Monitored monitored, long terminationTimeoutMs) {
+      super("Stopper", monitored::stop, terminationTimeoutMs);
+    }
+
+  }
+
+  /**
+   * Stops process in a short time fashion
+   */
+  private static class HardStopperThread extends AbstractStopperThread {
+
+    private HardStopperThread(Monitored monitored, long terminationTimeoutMs, @Nullable StopperThread stopperThread) {
+      super(
+        "HardStopper",
+        () -> {
+          if (stopperThread != null) {
+            stopperThread.stopIt();
+          }
+          monitored.hardStop();
+        },
+        terminationTimeoutMs);
+    }
+
+  }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/Stoppable.java b/server/sonar-process/src/main/java/org/sonar/process/Stoppable.java
index 7e189583aef..cadc0ba58b2 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/Stoppable.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/Stoppable.java
@@ -21,6 +21,8 @@ package org.sonar.process;
 
 public interface Stoppable {
 
+  void stopAsync();
+
   void hardStopAsync();
 
 }
diff --git a/server/sonar-process/src/test/java/org/sonar/process/HardStopWatcherTest.java b/server/sonar-process/src/test/java/org/sonar/process/AbstractStopWatcherTest.java
index c8806d05f21..8fbc9b2cefd 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/HardStopWatcherTest.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/AbstractStopWatcherTest.java
@@ -26,29 +26,33 @@ import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
 import org.sonar.process.sharedmemoryfile.ProcessCommands;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-public class HardStopWatcherTest {
+public class AbstractStopWatcherTest {
 
   @Rule
   public TestRule safeguardTimeout = new DisableOnDebug(Timeout.seconds(60));
 
   @Test
   public void stop_if_receive_command() throws Exception {
+
     ProcessCommands commands = mock(ProcessCommands.class);
     when(commands.askedForHardStop()).thenReturn(false, true);
     Stoppable stoppable = mock(Stoppable.class);
 
-    HardStopWatcher underTest = new HardStopWatcher(commands, stoppable, 1L);
+    AbstractStopWatcher underTest = new AbstractStopWatcher("TheThreadName",
+      stoppable::hardStopAsync, commands::askedForHardStop, 1L) {};
     underTest.start();
 
     while (underTest.isAlive()) {
       Thread.sleep(1L);
     }
     verify(stoppable).hardStopAsync();
+    assertThat(underTest.getName()).isEqualTo("TheThreadName");
   }
 
   @Test
@@ -57,7 +61,8 @@ public class HardStopWatcherTest {
     when(commands.askedForHardStop()).thenReturn(false);
     Stoppable stoppable = mock(Stoppable.class);
 
-    HardStopWatcher underTest = new HardStopWatcher(commands, stoppable, 1L);
+    AbstractStopWatcher underTest = new AbstractStopWatcher("TheThreadName",
+      stoppable::hardStopAsync, commands::askedForHardStop, 1L) {};
     underTest.start();
     underTest.interrupt();
 
@@ -65,5 +70,6 @@ public class HardStopWatcherTest {
       Thread.sleep(1L);
     }
     verify(stoppable, never()).hardStopAsync();
+    assertThat(underTest.getName()).isEqualTo("TheThreadName");
   }
 }
diff --git a/server/sonar-process/src/test/java/org/sonar/process/HardStopperThreadTest.java b/server/sonar-process/src/test/java/org/sonar/process/AbstractStopperThreadTest.java
index a53b8f8a23c..a3e4050a14d 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/HardStopperThreadTest.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/AbstractStopperThreadTest.java
@@ -19,42 +19,60 @@
  */
 package org.sonar.process;
 
+import java.util.concurrent.TimeUnit;
 import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.timeout;
 import static org.mockito.Mockito.verify;
 
-public class HardStopperThreadTest {
+public class AbstractStopperThreadTest {
   private Monitored monitored = mock(Monitored.class);
 
   @Test
   public void stop_in_a_timely_fashion() {
     // max stop timeout is 5 seconds, but test fails after 3 seconds
     // -> guarantees that stop is immediate
-    HardStopperThread stopper = new HardStopperThread(monitored, 5000L);
+    AbstractStopperThread stopper = new AbstractStopperThread("theThreadName", () -> monitored.hardStop(), 5000L){};
     stopper.start();
 
     verify(monitored, timeout(3000)).hardStop();
+    assertThat(stopper.getName()).isEqualTo("theThreadName");
   }
 
   @Test
   public void stop_timeout() {
-    doAnswer(new Answer() {
-      @Override
-      public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
-        Thread.sleep(10000L);
-        return null;
-      }
+    doAnswer(invocationOnMock -> {
+      await().atMost(10, TimeUnit.SECONDS).until(() -> false);
+      return null;
     }).when(monitored).hardStop();
 
     // max stop timeout is 100 milliseconds
-    HardStopperThread stopper = new HardStopperThread(monitored, 100L);
+    AbstractStopperThread stopper = new AbstractStopperThread("theThreadName", () -> monitored.hardStop(), 5000L){};
     stopper.start();
 
     verify(monitored, timeout(3000)).hardStop();
+    assertThat(stopper.getName()).isEqualTo("theThreadName");
+  }
+
+  @Test
+  public void stopIt_interrupts_worker() {
+    doAnswer(invocationOnMock -> {
+      await().atMost(10, TimeUnit.SECONDS).until(() -> false);
+      return null;
+    }).when(monitored).hardStop();
+
+    // max stop timeout is 100 milliseconds
+    AbstractStopperThread stopper = new AbstractStopperThread("theThreadName", () -> monitored.hardStop(), 5000L){};
+    stopper.start();
+
+    verify(monitored, timeout(3_000)).hardStop();
+
+    stopper.stopIt();
+    await().atMost(3, TimeUnit.SECONDS).until(() -> !stopper.isAlive());
+    assertThat(stopper.isAlive()).isFalse();
   }
 }
diff --git a/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java b/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java
index bd0689d3ed1..2cbfe3715a7 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java
@@ -19,7 +19,9 @@
  */
 package org.sonar.process;
 
+import java.io.File;
 import java.io.IOException;
+import java.util.Properties;
 import org.apache.commons.io.FileUtils;
 import org.junit.Rule;
 import org.junit.Test;
@@ -31,9 +33,6 @@ import org.sonar.process.Lifecycle.State;
 import org.sonar.process.sharedmemoryfile.ProcessCommands;
 import org.sonar.process.test.StandardProcess;
 
-import java.io.File;
-import java.util.Properties;
-
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
@@ -89,7 +88,7 @@ public class ProcessEntryPointTest {
   }
 
   @Test
-  public void launch_then_request_hard_stop() throws Exception {
+  public void launch_then_request_graceful_stop() throws Exception {
     Props props = createProps();
     final ProcessEntryPoint entryPoint = new ProcessEntryPoint(props, exit, commands);
     final StandardProcess process = new StandardProcess();
@@ -109,9 +108,36 @@ public class ProcessEntryPointTest {
 
     // requests for graceful stop -> waits until down
     // Should terminate before the timeout of 30s
+    entryPoint.stop();
+
+    assertThat(process.getState()).isEqualTo(State.STOPPED);
+    assertThat(process.wasHardStopped()).isEqualTo(false);
+  }
+
+  @Test
+  public void launch_then_request_hard_stop() throws Exception {
+    Props props = createProps();
+    final ProcessEntryPoint entryPoint = new ProcessEntryPoint(props, exit, commands);
+    final StandardProcess process = new StandardProcess();
+
+    Thread runner = new Thread() {
+      @Override
+      public void run() {
+        // starts and waits until terminated
+        entryPoint.launch(process);
+      }
+    };
+    runner.start();
+
+    while (process.getState() != State.STARTED) {
+      Thread.sleep(10L);
+    }
+
+    // requests for stop hardly waiting
     entryPoint.hardStop();
 
     assertThat(process.getState()).isEqualTo(State.STOPPED);
+    assertThat(process.wasHardStopped()).isEqualTo(true);
   }
 
   @Test
@@ -181,6 +207,11 @@ public class ProcessEntryPointTest {
     }
 
     @Override
+    public void stop() {
+
+    }
+
+    @Override
     public void hardStop() {
 
     }
@@ -204,6 +235,11 @@ public class ProcessEntryPointTest {
     }
 
     @Override
+    public void stop() {
+
+    }
+
+    @Override
     public void hardStop() {
 
     }
diff --git a/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java b/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java
index 7db02eff2f1..ef85fd90227 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java
@@ -26,6 +26,7 @@ import org.sonar.process.Lifecycle.State;
 public class StandardProcess implements Monitored {
 
   private State state = State.INIT;
+  private boolean hardStopped = false;
 
   private final Thread daemon = new Thread() {
     @Override
@@ -64,13 +65,21 @@ public class StandardProcess implements Monitored {
     }
   }
 
+  @Override
+  public void stop() {
+    state = State.STOPPING;
+    daemon.interrupt();
+    state = State.STOPPED;
+  }
+
   /**
    * Blocks until stopped
    */
   @Override
   public void hardStop() {
-    state = State.STOPPING;
+    state = State.HARD_STOPPING;
     daemon.interrupt();
+    hardStopped = true;
     state = State.STOPPED;
   }
 
@@ -78,6 +87,10 @@ public class StandardProcess implements Monitored {
     return state;
   }
 
+  public boolean wasHardStopped() {
+    return hardStopped;
+  }
+
   public static void main(String[] args) {
     ProcessEntryPoint entryPoint = ProcessEntryPoint.createForArguments(args);
     entryPoint.launch(new StandardProcess());
diff --git a/server/sonar-server/src/main/java/org/sonar/server/app/WebServer.java b/server/sonar-server/src/main/java/org/sonar/server/app/WebServer.java
index b893e6cd7c8..d09df3a6043 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/app/WebServer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/app/WebServer.java
@@ -71,6 +71,12 @@ public class WebServer implements Monitored {
   }
 
   @Override
+  public void stop() {
+    // hard stop is as graceful as stop for the WebServer
+    hardStop();
+  }
+
+  @Override
   public void hardStop() {
     tomcat.terminate();
   }