SONAR-12043 Hard stop if stop takes too long

server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java

@@ -38,6 +38,7 @@ import org.sonar.application.process.ManagedProcessHandler;
 import org.sonar.application.process.ManagedProcessLifecycle;
 import org.sonar.application.process.ProcessLifecycleListener;
 import org.sonar.process.ProcessId;
+import org.sonar.process.ProcessProperties;
 
 import static org.sonar.application.NodeLifecycle.State.HARD_STOPPING;
 import static org.sonar.application.NodeLifecycle.State.RESTARTING;
@@ -45,6 +46,7 @@ import static org.sonar.application.NodeLifecycle.State.STOPPED;
 import static org.sonar.application.NodeLifecycle.State.STOPPING;
 import static org.sonar.application.process.ManagedProcessHandler.Timeout.newTimeout;
 import static org.sonar.process.ProcessProperties.Property.CE_GRACEFUL_STOP_TIMEOUT;
+import static org.sonar.process.ProcessProperties.Property.WEB_GRACEFUL_STOP_TIMEOUT;
 import static org.sonar.process.ProcessProperties.parseTimeoutMs;
 
 public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, ProcessLifecycleListener, AppStateListener {
@@ -111,17 +113,23 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
       case ELASTICSEARCH:
         return HARD_STOP_TIMEOUT;
       case WEB_SERVER:
-        return newTimeout(10, TimeUnit.MINUTES);
+        return newTimeout(getStopTimeoutMs(settings, WEB_GRACEFUL_STOP_TIMEOUT), TimeUnit.MILLISECONDS);
       case COMPUTE_ENGINE:
-        String timeoutMs = settings.getValue(CE_GRACEFUL_STOP_TIMEOUT.getKey())
-          .orElse(CE_GRACEFUL_STOP_TIMEOUT.getDefaultValue());
-        return newTimeout(parseTimeoutMs(CE_GRACEFUL_STOP_TIMEOUT, timeoutMs), TimeUnit.MILLISECONDS);
+        return newTimeout(getStopTimeoutMs(settings, CE_GRACEFUL_STOP_TIMEOUT), TimeUnit.MILLISECONDS);
       case APP:
       default:
         throw new IllegalArgumentException("Unsupported processId " + processId);
     }
   }
 
+  private static long getStopTimeoutMs(AppSettings settings, ProcessProperties.Property property) {
+    String timeoutMs = settings.getValue(property.getKey())
+      .orElse(property.getDefaultValue());
+    // give some time to CE/Web to shutdown itself after "timeoutMs"
+    long gracePeriod = HARD_STOP_TIMEOUT.getUnit().toMillis(HARD_STOP_TIMEOUT.getDuration());
+    return parseTimeoutMs(property, timeoutMs) + gracePeriod;
+  }
+
   private void tryToStartAll() throws InterruptedException {
     tryToStartEs();
     tryToStartWeb();
@@ -250,7 +258,6 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
         " (current thread name is \"{}\")", Thread.currentThread().getName());
       Thread.currentThread().interrupt();
     }
-    awaitTermination.countDown();
   }
 
   private void hardStopAll() throws InterruptedException {
@@ -258,20 +265,22 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
     hardStopProcess(ProcessId.COMPUTE_ENGINE);
     hardStopProcess(ProcessId.WEB_SERVER);
     hardStopProcess(ProcessId.ELASTICSEARCH);
-
-    // if all process are already stopped (may occur, eg., when stopping because restart of 1st process failed),
-    // node state won't be updated on process stopped callback, so we must ensure
-    // the node's state is updated
-    if (nodeLifecycle.getState() != RESTARTING) {
-      nodeLifecycle.tryToMoveTo(STOPPED);
-    }
   }
 
+  /**
+   * This might be called twice: once by the state listener and once by the stop/hardStop implementations.
+   * The reason is that if all process are already stopped (may occur, eg., when stopping because restart of 1st process failed),
+   * the node state won't be updated on process stopped callback.
+   */
   private void finalizeStop() {
-    if (nodeLifecycle.getState() == STOPPED) {
+    if (nodeLifecycle.getState() != RESTARTING) {
       interrupt(restartStopperThread);
       interrupt(hardStopperThread);
       interrupt(restarterThread);
+      if (nodeLifecycle.tryToMoveTo(STOPPED)) {
+        LOG.info("SonarQube is stopped");
+      }
+      awaitTermination.countDown();
     }
   }
 
@@ -364,11 +373,9 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
         }
         break;
       case HARD_STOPPING:
-        if (lastProcessStopped && nodeLifecycle.tryToMoveTo(STOPPED)) {
-          LOG.info("SonarQube is stopped");
-          // all processes are stopped, no restart requested
-          // Let's clean-up resources
-          hardStop();
+      case STOPPING:
+        if (lastProcessStopped) {
+          finalizeStop();
         }
         break;
       default:
@@ -408,7 +415,7 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
   }
 
   private class RestarterThread extends Thread {
-    public RestarterThread() {
+    private RestarterThread() {
       super("Restarter");
     }
 
@@ -429,7 +436,7 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
   }
 
   private class RestartStopperThread extends Thread {
-    public RestartStopperThread() {
+    private RestartStopperThread() {
       super("Restart stopper");
     }
 
@@ -441,7 +448,7 @@ public class SchedulerImpl implements Scheduler, ManagedProcessEventListener, Pr
 
   private class HardStopperThread extends Thread {
 
-    public HardStopperThread() {
+    private HardStopperThread() {
       super("Hard stopper");
     }
 

server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java

@@ -162,7 +162,7 @@ public class CommandFactoryImpl implements CommandFactory {
       .setReadsArgumentsFromFile(true)
       .setArguments(props.rawProperties())
       .setJvmOptions(jvmOptions)
-      .setGracefulStopTimeoutMs(readTimeout(props, WEB_GRACEFUL_STOP_TIMEOUT))
+      .setGracefulStopTimeoutMs(getGracefulStopTimeoutMs(props, WEB_GRACEFUL_STOP_TIMEOUT))
       // required for logback tomcat valve
       .setEnvVariable(PATH_LOGS.getKey(), props.nonNullValue(PATH_LOGS.getKey()))
       .setArgument("sonar.cluster.web.startupLeader", Boolean.toString(leader))
@@ -189,7 +189,7 @@ public class CommandFactoryImpl implements CommandFactory {
       .setReadsArgumentsFromFile(true)
       .setArguments(props.rawProperties())
       .setJvmOptions(jvmOptions)
-      .setGracefulStopTimeoutMs(readTimeout(props, CE_GRACEFUL_STOP_TIMEOUT))
+      .setGracefulStopTimeoutMs(getGracefulStopTimeoutMs(props, CE_GRACEFUL_STOP_TIMEOUT))
       .setClassName("org.sonar.ce.app.CeServer")
       .addClasspath("./lib/common/*");
     String driverPath = props.value(JDBC_DRIVER_PATH.getKey());
@@ -200,10 +200,12 @@ public class CommandFactoryImpl implements CommandFactory {
     return command;
   }
 
-  private static long readTimeout(Props props, ProcessProperties.Property property) {
+  private static long getGracefulStopTimeoutMs(Props props, ProcessProperties.Property property) {
     String value = Optional.ofNullable(props.value(property.getKey()))
       .orElse(property.getDefaultValue());
-    return parseTimeoutMs(property, value);
+    // give some time to CE/Web to shutdown itself after graceful stop timed out
+    long gracePeriod = 30 * 1_000L;
+    return parseTimeoutMs(property, value) + gracePeriod;
   }
 
   private <T extends JvmOptions> void addProxyJvmOptions(JvmOptions<T> jvmOptions) {

server/sonar-main/src/main/java/org/sonar/application/command/EsJvmOptions.java

@@ -47,7 +47,7 @@ public class EsJvmOptions extends JvmOptions<EsJvmOptions> {
   // this basically writes down the content of jvm.options file distributed in vanilla Elasticsearch package
   // with some changes to fit running bundled in SQ
   private static Map<String, String> mandatoryOptions(System2 system2, Props props, File tmpDir) {
-    Map<String, String> res = new LinkedHashMap<>(16);
+    Map<String, String> res = new LinkedHashMap<>(30);
     // GC configuration
     res.put("-XX:+UseConcMarkSweepGC", "");
     res.put("-XX:CMSInitiatingOccupancyFraction=", "75");

server/sonar-main/src/main/java/org/sonar/application/process/AbstractManagedProcess.java

@@ -23,6 +23,7 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,7 +33,7 @@ abstract class AbstractManagedProcess implements ManagedProcess {
 
   private static final Logger LOG = LoggerFactory.getLogger(AbstractManagedProcess.class);
   private static final int EXPECTED_EXIT_VALUE = 0;
-
+  private final AtomicBoolean exitValueLogged = new AtomicBoolean(false);
   protected final Process process;
   private final ProcessId processId;
 
@@ -75,10 +76,12 @@ abstract class AbstractManagedProcess implements ManagedProcess {
 
   public void waitFor() throws InterruptedException {
     int exitValue = process.waitFor();
-    if (exitValue != EXPECTED_EXIT_VALUE) {
-      LOG.warn("Process exited with exit value [{}]: {}", processId.getKey(), exitValue);
-    } else {
-      LOG.debug("Process exited with exit value [{}]: {}", processId.getKey(), exitValue);
+    if (exitValueLogged.compareAndSet(false, true)) {
+      if (exitValue != EXPECTED_EXIT_VALUE) {
+        LOG.warn("Process exited with exit value [{}]: {}", processId.getKey(), exitValue);
+      } else {
+        LOG.debug("Process exited with exit value [{}]: {}", processId.getKey(), exitValue);
+      }
     }
   }
 

server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java

@@ -38,12 +38,10 @@ public class EsManagedProcess extends AbstractManagedProcess {
   private static final int WAIT_FOR_UP_DELAY_IN_MILLIS = 100;
   private static final int WAIT_FOR_UP_TIMEOUT = 10 * 60; /* 1min */
 
-  private final AtomicBoolean nodeUp = new AtomicBoolean(false);
-  private final AtomicBoolean nodeOperational = new AtomicBoolean(false);
+  private volatile boolean nodeOperational = false;
   private final AtomicBoolean firstMasterNotDiscoveredLog = new AtomicBoolean(true);
   private final EsConnector esConnector;
 
-
   public EsManagedProcess(Process process, ProcessId processId, EsConnector esConnector) {
     super(process, processId);
     this.esConnector = esConnector;
@@ -51,7 +49,7 @@ public class EsManagedProcess extends AbstractManagedProcess {
 
   @Override
   public boolean isOperational() {
-    if (nodeOperational.get()) {
+    if (nodeOperational) {
       return true;
     }
 
@@ -64,10 +62,10 @@ public class EsManagedProcess extends AbstractManagedProcess {
     } finally {
       if (flag) {
         esConnector.stop();
-        nodeOperational.set(true);
+        nodeOperational = true;
       }
     }
-    return nodeOperational.get();
+    return nodeOperational;
   }
 
   private boolean checkOperational() throws InterruptedException {
@@ -75,13 +73,13 @@ public class EsManagedProcess extends AbstractManagedProcess {
     Status status = checkStatus();
     do {
       if (status != Status.CONNECTION_REFUSED) {
-        nodeUp.set(true);
+        break;
       } else {
         Thread.sleep(WAIT_FOR_UP_DELAY_IN_MILLIS);
         i++;
         status = checkStatus();
       }
-    } while (!nodeUp.get() && i < WAIT_FOR_UP_TIMEOUT);
+    } while (i < WAIT_FOR_UP_TIMEOUT);
     return status == YELLOW || status == GREEN;
   }
 

server/sonar-main/src/main/java/org/sonar/application/process/ManagedProcessHandler.java

@@ -23,7 +23,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,7 +50,7 @@ public class ManagedProcessHandler {
   private final EventWatcher eventWatcher;
   // keep flag so that the operational event is sent only once
   // to listeners
-  private final AtomicBoolean operational = new AtomicBoolean(false);
+  private boolean operational = false;
 
   private ManagedProcessHandler(Builder builder) {
     this.processId = requireNonNull(builder.processId, "processId can't be null");
@@ -101,9 +100,11 @@ public class ManagedProcessHandler {
       stopImpl();
       if (process != null && process.isAlive()) {
         LOG.info("{} failed to stop in a graceful fashion. Hard stopping it.", processId.getKey());
+        hardStop();
+      } else {
+        // enforce stop and clean-up even if process has been quickly stopped
+        stopForcibly();
       }
-      // enforce stop and clean-up even if process has been quickly stopped
-      stopForcibly();
     } else {
       // already stopping or stopped
       waitForDown();
@@ -191,21 +192,20 @@ public class ManagedProcessHandler {
       StreamGobbler.waitUntilFinish(stdErrGobbler);
       stdErrGobbler.interrupt();
     }
+    // will trigger state listeners
     lifecycle.tryToMoveTo(ManagedProcessLifecycle.State.STOPPED);
   }
 
   void refreshState() {
     if (process.isAlive()) {
-      if (!operational.get() && process.isOperational()) {
-        operational.set(true);
+      if (!operational && process.isOperational()) {
+        operational = true;
         eventListeners.forEach(l -> l.onManagedProcessEvent(processId, ManagedProcessEventListener.Type.OPERATIONAL));
       }
       if (process.askedForRestart()) {
         process.acknowledgeAskForRestart();
         eventListeners.forEach(l -> l.onManagedProcessEvent(processId, ManagedProcessEventListener.Type.ASK_FOR_RESTART));
       }
-    } else {
-      stopForcibly();
     }
   }
 
@@ -218,9 +218,9 @@ public class ManagedProcessHandler {
    * This thread blocks as long as the monitored process is physically alive.
    * It avoids from executing {@link Process#exitValue()} at a fixed rate :
    * <ul>
-   *   <li>no usage of exception for flow control. Indeed {@link Process#exitValue()} throws an exception
-   *   if process is alive. There's no method <code>Process#isAlive()</code></li>
-   *   <li>no delay, instantaneous notification that process is down</li>
+   * <li>no usage of exception for flow control. Indeed {@link Process#exitValue()} throws an exception
+   * if process is alive. There's no method <code>Process#isAlive()</code></li>
+   * <li>no delay, instantaneous notification that process is down</li>
    * </ul>
    */
   private class StopWatcher extends Thread {
@@ -228,7 +228,7 @@ public class ManagedProcessHandler {
       // this name is different than Thread#toString(), which includes name, priority
       // and thread group
       // -> do not override toString()
-      super(format("HardStopWatcher[%s]", processId.getKey()));
+      super(format("StopWatcher[%s]", processId.getKey()));
     }
 
     @Override
@@ -259,10 +259,7 @@ public class ManagedProcessHandler {
           Thread.sleep(watcherDelayMs);
         }
       } catch (InterruptedException e) {
-        // request to stop watching process. To avoid unexpected behaviors
-        // the process is stopped.
         Thread.currentThread().interrupt();
-        stopForcibly();
       }
     }
   }

server/sonar-main/src/test/java/org/sonar/application/process/ManagedProcessHandlerTest.java

@@ -22,6 +22,7 @@ package org.sonar.application.process;
 import java.io.InputStream;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import org.awaitility.Awaitility;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.DisableOnDebug;
@@ -70,16 +71,16 @@ public class ManagedProcessHandlerTest {
       verify(listener).onProcessState(A_PROCESS_ID, ManagedProcessLifecycle.State.STARTED);
 
       testProcess.close();
-      // do not wait next run of watcher threads
-      underTest.refreshState();
-      assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.STOPPED);
+      Awaitility.await()
+        .atMost(10, TimeUnit.SECONDS)
+        .until(() -> underTest.getState() == ManagedProcessLifecycle.State.STOPPED);
       assertThat(testProcess.isAlive()).isFalse();
       assertThat(testProcess.streamsClosed).isTrue();
       verify(listener).onProcessState(A_PROCESS_ID, ManagedProcessLifecycle.State.STOPPED);
     }
   }
 
-  public ManagedProcessHandler.Builder newHanderBuilder(ProcessId aProcessId) {
+  private ManagedProcessHandler.Builder newHanderBuilder(ProcessId aProcessId) {
     return ManagedProcessHandler.builder(aProcessId)
       .setStopTimeout(newTimeout(1, TimeUnit.SECONDS))
       .setHardStopTimeout(newTimeout(1, TimeUnit.SECONDS));
@@ -232,7 +233,31 @@ public class ManagedProcessHandlerTest {
   }
 
   @Test
-  public void process_is_stopped_forcibly_if_graceful_stop_is_too_long() throws Exception {
+  public void process_is_hard_stopped_if_graceful_stop_is_too_long() throws Exception {
+    ProcessLifecycleListener listener = mock(ProcessLifecycleListener.class);
+    ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
+      .addProcessLifecycleListener(listener)
+      .setStopTimeout(newTimeout(1, TimeUnit.MILLISECONDS))
+      .setHardStopTimeout(newTimeout(1, TimeUnit.MILLISECONDS))
+      .build();
+
+    try (TestManagedProcess testProcess = new TestManagedProcess()) {
+      underTest.start(() -> testProcess);
+
+      underTest.stop();
+
+      testProcess.waitFor();
+      assertThat(testProcess.askedForHardStop).isTrue();
+      assertThat(testProcess.askedForStop).isTrue();
+      assertThat(testProcess.destroyedForcibly).isTrue();
+      assertThat(testProcess.isAlive()).isFalse();
+      assertThat(underTest.getState()).isEqualTo(ManagedProcessLifecycle.State.STOPPED);
+      verify(listener).onProcessState(A_PROCESS_ID, ManagedProcessLifecycle.State.STOPPED);
+    }
+  }
+
+  @Test
+  public void process_is_stopped_forcibly_if_hard_stop_is_too_long() throws Exception {
     ProcessLifecycleListener listener = mock(ProcessLifecycleListener.class);
     ManagedProcessHandler underTest = newHanderBuilder(A_PROCESS_ID)
       .addProcessLifecycleListener(listener)

server/sonar-process/src/main/java/org/sonar/process/AbstractStopperThread.java

@@ -19,10 +19,13 @@
  */
 package org.sonar.process;
 
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import org.slf4j.LoggerFactory;
 
 /**
@@ -32,7 +35,6 @@ abstract class AbstractStopperThread extends Thread {
 
   private final Runnable stopCode;
   private final long terminationTimeoutMs;
-  private boolean stop = false;
 
   AbstractStopperThread(String threadName, Runnable stopCode, long terminationTimeoutMs) {
     super(threadName);
@@ -43,20 +45,23 @@ abstract class AbstractStopperThread extends Thread {
 
   @Override
   public void run() {
-    ExecutorService executor = Executors.newSingleThreadExecutor();
+    ExecutorService executor = Executors.newSingleThreadExecutor(
+      new ThreadFactoryBuilder()
+        .setDaemon(false)
+        .setNameFormat(getName() + "-%d")
+        .build());
     try {
       Future future = executor.submit(stopCode);
       future.get(terminationTimeoutMs, TimeUnit.MILLISECONDS);
-    } catch (Exception e) {
-      if (!stop) {
-        LoggerFactory.getLogger(getClass()).error("Can not stop in {}ms", terminationTimeoutMs, e);
-      }
+    } catch (TimeoutException | InterruptedException e) {
+      LoggerFactory.getLogger(getClass()).warn("Can not stop in {}ms", terminationTimeoutMs);
+    } catch (ExecutionException e) {
+      LoggerFactory.getLogger(getClass()).error("Can not stop in {}ms", terminationTimeoutMs, e);
     }
     executor.shutdownNow();
   }
 
   public void stopIt() {
-    this.stop = true;
     super.interrupt();
   }
 }

server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java

@@ -20,7 +20,6 @@
 package org.sonar.process;
 
 import java.io.File;
-import java.util.concurrent.CountDownLatch;
 import java.util.function.Predicate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -82,8 +81,6 @@ public class ProcessEntryPoint {
       launch(logger);
     } catch (Exception e) {
       logger.warn("Fail to start {}", processKey, e);
-    } finally {
-      logger.trace("Hard stopping to clean any resource...");
       hardStop();
     }
   }
@@ -163,6 +160,8 @@ public class ProcessEntryPoint {
 
   private void terminate() {
     lifecycle.tryToMoveTo(STOPPED);
+    hardStopWatcher.stopWatching();
+    stopWatcher.stopWatching();
     commands.endWatch();
   }
 

server/sonar-process/src/main/java/org/sonar/process/StopWatcher.java

@@ -21,8 +21,11 @@ package org.sonar.process;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.util.function.BooleanSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class StopWatcher extends Thread {
+  private static final Logger LOG = LoggerFactory.getLogger(StopWatcher.class);
   private final Runnable stopCommand;
   private final BooleanSupplier shouldStopTest;
   private final long delayMs;
@@ -44,6 +47,7 @@ public class StopWatcher extends Thread {
   public void run() {
     while (watching) {
       if (shouldStopTest.getAsBoolean()) {
+        LOG.trace("{} triggering stop command", this.getName());
         stopCommand.run();
         watching = false;
       } else {

server/sonar-server/src/main/java/org/sonar/server/app/EmbeddedTomcat.java

@@ -21,6 +21,7 @@ package org.sonar.server.app;
 
 import com.google.common.base.Throwables;
 import java.io.File;
+import java.util.concurrent.CountDownLatch;
 import org.apache.catalina.LifecycleException;
 import org.apache.catalina.core.StandardContext;
 import org.apache.catalina.startup.Tomcat;
@@ -35,6 +36,7 @@ class EmbeddedTomcat {
   private final Props props;
   private Tomcat tomcat = null;
   private volatile StandardContext webappContext;
+  private final CountDownLatch stopLatch = new CountDownLatch(1);
 
   EmbeddedTomcat(Props props) {
     this.props = props;
@@ -94,18 +96,27 @@ class EmbeddedTomcat {
   }
 
   void terminate() {
-    if (tomcat.getServer().getState().isAvailable()) {
-      try {
-        tomcat.stop();
-        tomcat.destroy();
-      } catch (Exception e) {
-        Loggers.get(EmbeddedTomcat.class).error("Fail to stop web server", e);
+    try {
+      if (tomcat.getServer().getState().isAvailable()) {
+        try {
+          tomcat.stop();
+          tomcat.destroy();
+        } catch (Exception e) {
+          Loggers.get(EmbeddedTomcat.class).warn("Failed to stop web server", e);
+        }
       }
+      deleteQuietly(tomcatBasedir());
+    } finally {
+      stopLatch.countDown();
     }
-    deleteQuietly(tomcatBasedir());
   }
 
   void awaitTermination() {
-    tomcat.getServer().await();
+    try {
+      // calling tomcat.getServer().await() might block forever if stop fails for whatever reason
+      stopLatch.await();
+    } catch (InterruptedException e) {
+      // quit
+    }
   }
 }

server/sonar-server/src/main/java/org/sonar/server/app/WebServer.java

@@ -21,12 +21,12 @@ package org.sonar.server.app;
 
 import com.google.common.collect.ImmutableMap;
 import java.io.File;
-import org.sonar.process.sharedmemoryfile.DefaultProcessCommands;
 import org.sonar.process.MinimumViableSystem;
 import org.sonar.process.Monitored;
 import org.sonar.process.ProcessEntryPoint;
 import org.sonar.process.ProcessId;
 import org.sonar.process.Props;
+import org.sonar.process.sharedmemoryfile.DefaultProcessCommands;
 
 public class WebServer implements Monitored {
   public static final String PROPERTY_SHARED_PATH = "process.sharedDir";

server/sonar-server/src/main/java/org/sonar/server/platform/web/PlatformServletContextListener.java

@@ -19,13 +19,11 @@
  */
 package org.sonar.server.platform.web;
 
-import ch.qos.logback.classic.LoggerContext;
 import java.util.Enumeration;
 import java.util.Properties;
 import javax.servlet.ServletContext;
 import javax.servlet.ServletContextEvent;
 import javax.servlet.ServletContextListener;
-import org.slf4j.LoggerFactory;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.server.platform.Platform;
 
@@ -66,8 +64,6 @@ public final class PlatformServletContextListener implements ServletContextListe
   @Override
   public void contextDestroyed(ServletContextEvent event) {
     Platform.getInstance().doStop();
-    LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
-    loggerContext.stop();
   }
 
 }