SONAR-4898 fix handling of server early termination

server/sonar-process/src/main/java/org/sonar/process/JmxUtils.java

@@ -30,6 +30,9 @@ public class JmxUtils {
     // only static stuff
   }
 
+  public static final String WEB_SERVER_NAME = "web";
+  public static final String SEARCH_SERVER_NAME = "search";
+
   public static ObjectName objectName(String name) {
     try {
       return new ObjectName("org.sonar", "name", name);

server/sonar-process/src/main/java/org/sonar/process/Monitor.java

@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
@@ -38,7 +39,7 @@ public class Monitor extends Thread implements Terminable {
   private final static Logger LOGGER = LoggerFactory.getLogger(Monitor.class);
 
   private volatile List<ProcessWrapper> processes;
-  private volatile Map<String, Long> pings;
+  private final Map<String, Long> pings;
   private final ScheduledFuture<?> watch;
   private final ScheduledExecutorService monitor;
 
@@ -48,7 +49,7 @@ public class Monitor extends Thread implements Terminable {
   public Monitor() {
     super("Process Monitor");
     processes = new ArrayList<ProcessWrapper>();
-    pings = new HashMap<String, Long>();
+    pings = new ConcurrentHashMap<String, Long>();
     monitor = Executors.newScheduledThreadPool(1);
     watch = monitor.scheduleWithFixedDelay(new ProcessWatch(), 0L, PING_DELAY_MS, TimeUnit.MILLISECONDS);
   }
@@ -99,14 +100,17 @@ public class Monitor extends Thread implements Terminable {
   @Override
   public void run() {
     try {
-      while (true) {
+      boolean ok = true;
+      while (ok) {
         for (ProcessWrapper process : processes) {
           if (!processIsValid(process)) {
-            LOGGER.warn("Monitor::run() -- Process '{}' is not valid. Exiting monitor", process.getName());
+            ok = false;
             interrupt();
           }
         }
-        Thread.sleep(PING_DELAY_MS);
+        if (ok) {
+          Thread.sleep(PING_DELAY_MS);
+        }
       }
     } catch (InterruptedException e) {
       LOGGER.debug("Monitoring thread is interrupted");
@@ -117,12 +121,17 @@ public class Monitor extends Thread implements Terminable {
 
   @Override
   public void terminate() {
-    processes.clear();
     if (!monitor.isShutdown()) {
       monitor.shutdownNow();
     }
     if (!watch.isCancelled()) {
       watch.cancel(true);
     }
+
+    for (int i=processes.size()-1 ; i>=0 ; i--) {
+      processes.get(i).terminate();
+    }
+    processes.clear();
+    interrupt();
   }
 }

server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java

@@ -176,9 +176,7 @@ public class ProcessWrapper extends Thread implements Terminable {
       waitUntilFinish(errorGobbler);
       ProcessUtils.closeStreams(process);
       FileUtils.deleteQuietly(propertiesFile);
-      processMXBean = null;
     }
-    LOGGER.trace("ProcessWrapper::run() END");
   }
 
   public boolean isReady() {
@@ -261,7 +259,7 @@ public class ProcessWrapper extends Thread implements Terminable {
 
   @Override
   public void terminate() {
-    if (processMXBean != null) {
+    if (processMXBean != null && process != null) {
       // Send the terminate command to process in order to gracefully shutdown.
       // Then hardly kill it if it didn't terminate in 30 seconds
       ScheduledExecutorService killer = Executors.newScheduledThreadPool(1);
@@ -280,6 +278,7 @@ public class ProcessWrapper extends Thread implements Terminable {
 
       } catch (Exception ignored) {
         // ignore
+        ignored.printStackTrace();
       } finally {
         killer.shutdownNow();
       }
@@ -287,6 +286,7 @@ public class ProcessWrapper extends Thread implements Terminable {
       // process is not monitored through JMX, but killing it though
       ProcessUtils.destroyQuietly(process);
     }
+    processMXBean = null;
   }
 
   public boolean waitForReady() throws InterruptedException {
@@ -297,6 +297,9 @@ public class ProcessWrapper extends Thread implements Terminable {
     long wait = 500L;
     while (now < READY_TIMEOUT_MS) {
       try {
+        if (processMXBean == null) {
+          return false;
+        }
         if (processMXBean.isReady()) {
           return true;
         }

server/sonar-server/src/main/java/org/sonar/server/app/EmbeddedTomcat.java

@@ -20,6 +20,7 @@
 package org.sonar.server.app;
 
 import org.apache.catalina.connector.Connector;
+import org.apache.catalina.core.StandardContext;
 import org.apache.catalina.startup.Tomcat;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.LoggerFactory;
@@ -34,7 +35,7 @@ class EmbeddedTomcat implements Terminable {
   private final Props props;
   private Tomcat tomcat = null;
   private Thread hook = null;
-  private boolean stopping = false, ready = false;
+  private boolean ready = false;
 
   EmbeddedTomcat(Props props) {
     this.props = props;
@@ -65,11 +66,14 @@ class EmbeddedTomcat implements Terminable {
 
       Logging.configure(tomcat, props);
       Connectors.configure(tomcat, props);
-      Webapp.configure(tomcat, props);
+      StandardContext webappContext = Webapp.configure(tomcat, props);
       ProcessUtils.addSelfShutdownHook(this);
       tomcat.start();
-      ready = true;
-      tomcat.getServer().await();
+
+      if (webappContext.getState().isAvailable()) {
+        ready = true;
+        tomcat.getServer().await();
+      }
     } catch (Exception e) {
       throw new IllegalStateException("Fail to start web server", e);
     } finally {
@@ -96,17 +100,18 @@ class EmbeddedTomcat implements Terminable {
 
   @Override
   public void terminate() {
-    if (tomcat != null && !stopping) {
-      try {
-        stopping = true;
-        tomcat.stop();
-        tomcat.destroy();
-      } catch (Exception e) {
-        LoggerFactory.getLogger(EmbeddedTomcat.class).error("Fail to stop web service", e);
+    if (tomcat != null) {
+      synchronized (tomcat) {
+        if (tomcat.getServer().getState().isAvailable()) {
+          try {
+            tomcat.stop();
+            tomcat.destroy();
+          } catch (Exception e) {
+            LoggerFactory.getLogger(EmbeddedTomcat.class).error("Fail to stop web service", e);
+          }
+        }
       }
     }
-    tomcat = null;
-    stopping = false;
     ready = false;
     FileUtils.deleteQuietly(tomcatBasedir());
   }

server/sonar-server/src/main/java/org/sonar/server/app/Webapp.java

@@ -33,7 +33,7 @@ class Webapp {
   private static final String RAILS_ENV = "rails.env";
   private static final String PROPERTY_CONTEXT = "sonar.web.context";
 
-  static void configure(Tomcat tomcat, Props props) {
+  static StandardContext configure(Tomcat tomcat, Props props) {
     try {
       String webDir = props.of("sonar.path.web");
       LoggerFactory.getLogger(Webapp.class).info("Webapp directory: " + webDir);
@@ -55,6 +55,7 @@ class Webapp {
       }
       configureRailsMode(props, context);
       context.setJarScanner(new NullJarScanner());
+      return context;
 
     } catch (Exception e) {
       throw new IllegalStateException("Fail to configure webapp", e);

server/sonar-server/src/main/java/org/sonar/server/platform/PlatformServletContextListener.java

@@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory;
 import javax.servlet.ServletContext;
 import javax.servlet.ServletContextEvent;
 import javax.servlet.ServletContextListener;
+
 import java.util.Enumeration;
 import java.util.Properties;
 
@@ -49,7 +50,7 @@ public final class PlatformServletContextListener implements ServletContextListe
       // unexpected errors
       LoggerFactory.getLogger(getClass()).error("Fail to start server", t);
       stopQuietly();
-      throw new IllegalStateException("Fail to start server", t);
+      throw new IllegalStateException("Fail to start webapp", t);
     }
   }
 

server/sonar-server/src/main/java/org/sonar/server/plugins/ServerPluginJarsInstaller.java

@@ -20,7 +20,6 @@
 package org.sonar.server.plugins;
 
 import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.io.FileUtils;
@@ -53,7 +52,7 @@ public class ServerPluginJarsInstaller {
   private final ServerUpgradeStatus serverUpgradeStatus;
 
   public ServerPluginJarsInstaller(Server server, ServerUpgradeStatus serverUpgradeStatus,
-                                   DefaultServerFileSystem fs, ServerPluginJarInstaller installer) {
+    DefaultServerFileSystem fs, ServerPluginJarInstaller installer) {
     this.server = server;
     this.serverUpgradeStatus = serverUpgradeStatus;
     this.fs = fs;
@@ -97,7 +96,7 @@ public class ServerPluginJarsInstaller {
 
   private void moveDownloadedPlugins() {
     if (fs.getDownloadedPluginsDir().exists()) {
-      Collection<File> sourceFiles = FileUtils.listFiles(fs.getDownloadedPluginsDir(), new String[]{"jar"}, false);
+      Collection<File> sourceFiles = FileUtils.listFiles(fs.getDownloadedPluginsDir(), new String[] {"jar"}, false);
       for (File sourceFile : sourceFiles) {
         overridePlugin(sourceFile, true);
       }
@@ -117,7 +116,6 @@ public class ServerPluginJarsInstaller {
     }
   }
 
-
   private void overridePlugin(File sourceFile, boolean deleteSource) {
     File destDir = fs.getUserPluginsDir();
     File destFile = new File(destDir, sourceFile.getName());
@@ -181,7 +179,7 @@ public class ServerPluginJarsInstaller {
   public List<String> getUninstalls() {
     List<String> names = Lists.newArrayList();
     if (fs.getTrashPluginsDir().exists()) {
-      List<File> files = (List<File>) FileUtils.listFiles(fs.getTrashPluginsDir(), new String[]{"jar"}, false);
+      List<File> files = (List<File>) FileUtils.listFiles(fs.getTrashPluginsDir(), new String[] {"jar"}, false);
       for (File file : files) {
         names.add(file.getName());
       }
@@ -191,7 +189,7 @@ public class ServerPluginJarsInstaller {
 
   public void cancelUninstalls() {
     if (fs.getTrashPluginsDir().exists()) {
-      List<File> files = (List<File>) FileUtils.listFiles(fs.getTrashPluginsDir(), new String[]{"jar"}, false);
+      List<File> files = (List<File>) FileUtils.listFiles(fs.getTrashPluginsDir(), new String[] {"jar"}, false);
       for (File file : files) {
         try {
           FileUtils.moveFileToDirectory(file, fs.getUserPluginsDir(), false);
@@ -211,9 +209,10 @@ public class ServerPluginJarsInstaller {
   private void deploy(DefaultPluginMetadata plugin) {
     LOG.info("Deploy plugin {}", Joiner.on(" / ").skipNulls().join(plugin.getName(), plugin.getVersion(), plugin.getImplementationBuild()));
 
-    Preconditions.checkState(plugin.isCompatibleWith(server.getVersion()),
-      "Plugin %s needs a more recent version of SonarQube than %s. At least %s is expected",
-      plugin.getKey(), server.getVersion(), plugin.getSonarVersion());
+    if (!plugin.isCompatibleWith(server.getVersion())) {
+      throw MessageException.of(String.format("Plugin %s needs a more recent version of SonarQube than %s. At least %s is expected",
+        plugin.getKey(), server.getVersion(), plugin.getSonarVersion()));
+    }
 
     try {
       File pluginDeployDir = new File(fs.getDeployedPluginsDir(), plugin.getKey());

sonar-application/src/main/java/org/sonar/application/App.java

@@ -29,9 +29,6 @@ import org.sonar.process.ProcessWrapper;
 
 public class App implements ProcessMXBean {
 
-  static final String SONAR_WEB_PROCESS = "web";
-  static final String SONAR_SEARCH_PROCESS = "search";
-
   private final Installation installation;
 
   private final Monitor monitor = new Monitor();
@@ -49,7 +46,7 @@ public class App implements ProcessMXBean {
       Logger logger = LoggerFactory.getLogger(getClass());
       monitor.start();
 
-      elasticsearch = new ProcessWrapper(SONAR_SEARCH_PROCESS)
+      elasticsearch = new ProcessWrapper(JmxUtils.SEARCH_SERVER_NAME)
         .setWorkDir(installation.homeDir())
         .setJmxPort(Integer.parseInt(installation.prop(DefaultSettings.ES_JMX_PORT_KEY)))
         .addJavaOpts(installation.prop(DefaultSettings.ES_JAVA_OPTS_KEY))
@@ -64,7 +61,7 @@ public class App implements ProcessMXBean {
         if (elasticsearch.waitForReady()) {
           logger.info("Search server is ready");
 
-          server = new ProcessWrapper(SONAR_WEB_PROCESS)
+          server = new ProcessWrapper(JmxUtils.WEB_SERVER_NAME)
             .setWorkDir(installation.homeDir())
             .setJmxPort(Integer.parseInt(installation.prop(DefaultSettings.WEB_JMX_PORT_KEY)))
             .addJavaOpts(installation.prop(DefaultSettings.WEB_JAVA_OPTS_KEY))
@@ -105,8 +102,11 @@ public class App implements ProcessMXBean {
 
   @Override
   public void terminate() {
-    monitor.terminate();
-    monitor.interrupt();
+    LoggerFactory.getLogger(App.class).info("Stopping");
+    if (monitor.isAlive()) {
+      monitor.terminate();
+      monitor.interrupt();
+    }
     if (server != null) {
       server.terminate();
     }