SONAR-4898 improve reliability

fork.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-mvn clean install -DskipTests -Denforcer.skip=true -pl server/sonar-search,server/sonar-process,sonar-application
+mvn clean install -DskipTests -Denforcer.skip=true -pl :sonar-search,:sonar-process -amd
 
 if [[ "$OSTYPE" == "darwin"* ]]; then
   OS='macosx-universal-64'

server/sonar-process/src/main/java/org/sonar/process/JmxUtils.java

@@ -0,0 +1,51 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.process;
+
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+
+import java.lang.management.ManagementFactory;
+
+public class JmxUtils {
+  private JmxUtils() {
+    // only static stuff
+  }
+
+  public static ObjectName objectName(String name) {
+    try {
+      return new ObjectName("org.sonar", "name", name);
+    } catch (MalformedObjectNameException e) {
+      throw new IllegalStateException("Cannot create ObjectName for " + name, e);
+    }
+  }
+
+  public static void registerMBean(Object mbean, String name) {
+    try {
+      MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
+      mbeanServer.registerMBean(mbean, objectName(name));
+    } catch (RuntimeException re) {
+      throw re;
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to register JMX MBean named " + name, e);
+    }
+  }
+}

server/sonar-process/src/main/java/org/sonar/process/Monitor.java

@@ -31,89 +31,98 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
-public class Monitor extends Thread {
-
-  private static final long MAX_TIME = 15000L;
+public class Monitor extends Thread implements Terminatable {
 
+  private static final long PING_DELAY_MS = 3000L;
+  private static final long TIMEOUT_MS = 15000L;
   private final static Logger LOGGER = LoggerFactory.getLogger(Monitor.class);
 
   private volatile List<ProcessWrapper> processes;
   private volatile Map<String, Long> pings;
+  private final ScheduledFuture<?> watch;
+  private final ScheduledExecutorService monitor;
 
-  private ProcessWatch processWatch;
-  private ScheduledFuture<?> watch;
-  private ScheduledExecutorService monitor;
-
+  /**
+   * Starts another thread to send ping to all registered processes
+   */
   public Monitor() {
+    super("Process Monitor");
     processes = new ArrayList<ProcessWrapper>();
     pings = new HashMap<String, Long>();
     monitor = Executors.newScheduledThreadPool(1);
-    processWatch = new ProcessWatch();
-    watch = monitor.scheduleWithFixedDelay(processWatch, 0, 3, TimeUnit.SECONDS);
+    watch = monitor.scheduleWithFixedDelay(new ProcessWatch(), 0L, PING_DELAY_MS, TimeUnit.MILLISECONDS);
   }
 
-  public void registerProcess(ProcessWrapper processWrapper) {
-    processes.add(processWrapper);
-    pings.put(processWrapper.getName(), System.currentTimeMillis());
-    for (int i = 0; i < 10; i++) {
-      if (processWrapper.getProcessMXBean() == null || !processWrapper.getProcessMXBean().isReady()) {
-        try {
-          Thread.sleep(500L);
-        } catch (InterruptedException e) {
-          throw new IllegalStateException("Could not register process in Monitor", e);
-        }
-      }
+  private class ProcessWatch extends Thread {
+    private ProcessWatch() {
+      super("Process Ping");
     }
-    processWrapper.start();
-  }
 
-  private class ProcessWatch implements Runnable {
+    @Override
     public void run() {
       for (ProcessWrapper process : processes) {
         try {
-          if (process.getProcessMXBean() != null) {
-            long time = process.getProcessMXBean().ping();
-            LOGGER.debug("PINGED '{}'", process.getName());
+          ProcessMXBean mBean = process.getProcessMXBean();
+          if (mBean != null) {
+            long time = mBean.ping();
             pings.put(process.getName(), time);
           }
         } catch (Exception e) {
-          LOGGER.error("Error while pinging {}", process.getName(), e);
-          terminate();
+          // fail to ping, do nothing
         }
       }
     }
   }
 
-  private boolean processIsValid(ProcessWrapper process) {
-    long now = System.currentTimeMillis();
-    return (now - pings.get(process.getName())) < MAX_TIME;
+  /**
+   * Registers and monitors process. Note that process is probably not ready yet.
+   */
+  public void registerProcess(ProcessWrapper process) throws InterruptedException {
+    processes.add(process);
+    pings.put(process.getName(), System.currentTimeMillis());
+    // starts a monitoring thread
+    process.start();
   }
 
+  private boolean processIsValid(ProcessWrapper processWrapper) {
+    if (ProcessUtils.isAlive(processWrapper.process())) {
+      long now = System.currentTimeMillis();
+      return now - pings.get(processWrapper.getName()) < TIMEOUT_MS;
+    }
+    return false;
+  }
+
+  /**
+   * Check continuously that registered processes are still up. If any process is down or does not answer to pings
+   * during the max allowed period, then thread exits. 
+   */
+  @Override
   public void run() {
     try {
       while (true) {
         for (ProcessWrapper process : processes) {
           if (!processIsValid(process)) {
             LOGGER.warn("Monitor::run() -- Process '{}' is not valid. Exiting monitor", process.getName());
-            this.interrupt();
+            interrupt();
           }
         }
-        Thread.sleep(3000L);
+        Thread.sleep(PING_DELAY_MS);
       }
     } catch (InterruptedException e) {
-      LOGGER.debug("Monitoring thread is interrupted.");
+      LOGGER.debug("Monitoring thread is interrupted");
     } finally {
       terminate();
     }
   }
 
+  @Override
   public void terminate() {
-    if (monitor != null) {
+    processes.clear();
+    if (!monitor.isShutdown()) {
       monitor.shutdownNow();
+    }
+    if (!watch.isCancelled()) {
       watch.cancel(true);
-      watch = null;
-      processWatch = null;
     }
   }
-
 }

server/sonar-process/src/main/java/org/sonar/process/MonitoredProcess.java

@@ -0,0 +1,126 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.process;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+public abstract class MonitoredProcess implements ProcessMXBean {
+
+  public static final String NAME_PROPERTY = "pName";
+  private static final long AUTOKILL_TIMEOUT_MS = 15000L;
+  private static final long AUTOKILL_CHECK_DELAY_MS = 5000L;
+  public static final String MISSING_NAME_ARGUMENT = "Missing Name argument";
+
+  private Long lastPing;
+  private final String name;
+  protected final Props props;
+  private ScheduledFuture<?> pingTask = null;
+  private ScheduledExecutorService monitor;
+
+  protected MonitoredProcess(Props props) throws Exception {
+    this.props = props;
+    this.name = props.of(NAME_PROPERTY);
+
+    // Testing required properties
+    if (StringUtils.isEmpty(name)) {
+      throw new IllegalStateException(MISSING_NAME_ARGUMENT);
+    }
+
+    JmxUtils.registerMBean(this, name);
+    ProcessUtils.addSelfShutdownHook(this);
+  }
+
+  public final void start() {
+    if (monitor != null) {
+      throw new IllegalStateException("Already started");
+    }
+
+    Logger logger = LoggerFactory.getLogger(getClass());
+    logger.debug("Process[{}] starting", name);
+    scheduleAutokill();
+    doStart();
+    logger.debug("Process[{}] started", name);
+  }
+
+  /**
+   * If the process does not receive pings during the max allowed period, then
+   * process auto-kills
+   */
+  private void scheduleAutokill() {
+    final Runnable breakOnMissingPing = new Runnable() {
+      @Override
+      public void run() {
+        long time = System.currentTimeMillis();
+        if (time - lastPing > AUTOKILL_TIMEOUT_MS) {
+          terminate();
+        }
+      }
+    };
+    lastPing = System.currentTimeMillis();
+    monitor = Executors.newScheduledThreadPool(1);
+    pingTask = monitor.scheduleWithFixedDelay(breakOnMissingPing, AUTOKILL_CHECK_DELAY_MS, AUTOKILL_CHECK_DELAY_MS, TimeUnit.MILLISECONDS);
+  }
+
+  @Override
+  public final long ping() {
+    this.lastPing = System.currentTimeMillis();
+    return lastPing;
+  }
+
+  @Override
+  public final void terminate() {
+    if (monitor != null) {
+      monitor.shutdownNow();
+      monitor = null;
+      if (pingTask != null) {
+        pingTask.cancel(true);
+        pingTask = null;
+      }
+      try {
+        doTerminate();
+      } catch (Exception e) {
+        LoggerFactory.getLogger(getClass()).error("Fail to terminate " + name, e);
+        // do not propagate exception
+      }
+    }
+  }
+
+  @Override
+  public final boolean isReady() {
+    try {
+      return doIsReady();
+    } catch (Exception ignored) {
+      return false;
+    }
+  }
+
+  protected abstract void doStart();
+
+  protected abstract void doTerminate();
+
+  protected abstract boolean doIsReady();
+}

server/sonar-process/src/main/java/org/sonar/process/Process.java

@@ -1,143 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.process;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.InstanceAlreadyExistsException;
-import javax.management.MBeanRegistrationException;
-import javax.management.MBeanServer;
-import javax.management.MalformedObjectNameException;
-import javax.management.NotCompliantMBeanException;
-import javax.management.ObjectName;
-import java.lang.management.ManagementFactory;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-public abstract class Process implements ProcessMXBean {
-
-  public static final String NAME_PROPERTY = "pName";
-  public static final String PORT_PROPERTY = "pPort";
-  public static final String MISSING_NAME_ARGUMENT = "Missing Name argument";
-
-  private Long lastPing;
-  private String name;
-  private Integer port;
-
-  protected final Props props;
-
-  private static final long MAX_ALLOWED_TIME = 15000L;
-  private ScheduledFuture<?> pingTask = null;
-  private ScheduledExecutorService monitor = Executors.newScheduledThreadPool(1);
-  final Runnable breakOnMissingPing = new Runnable() {
-    public void run() {
-      long time = System.currentTimeMillis();
-      if (time - lastPing > MAX_ALLOWED_TIME) {
-        LoggerFactory.getLogger(getClass()).warn("Did not get a check-in for {}ms. Initiate shutdown", time - lastPing);
-        terminate();
-      }
-    }
-  };
-
-  protected Process(Props props) {
-    this.props = props;
-    init();
-  }
-
-  private void init() {
-    this.name = props.of(NAME_PROPERTY, null);
-    this.port = props.intOf(PORT_PROPERTY);
-
-    // Testing required properties
-    if (StringUtils.isEmpty(this.name)) {
-      throw new IllegalStateException(MISSING_NAME_ARGUMENT);
-    }
-
-    MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
-    try {
-      mbeanServer.registerMBean(this, this.getObjectName());
-    } catch (InstanceAlreadyExistsException e) {
-      throw new IllegalStateException("Process already exists in current JVM", e);
-    } catch (MBeanRegistrationException e) {
-      throw new IllegalStateException("Could not register process as MBean", e);
-    } catch (NotCompliantMBeanException e) {
-      throw new IllegalStateException("Process is not a compliant MBean", e);
-    }
-
-    Thread shutdownHook = new Thread(new Runnable() {
-      @Override
-      public void run() {
-        terminate();
-      }
-    });
-    Runtime.getRuntime().addShutdownHook(shutdownHook);
-  }
-
-  public ObjectName getObjectName() {
-    return objectNameFor(name);
-  }
-
-  static public ObjectName objectNameFor(String name) {
-    try {
-      return new ObjectName("org.sonar", "name", name);
-    } catch (MalformedObjectNameException e) {
-      throw new IllegalStateException("Cannot create ObjectName for " + name, e);
-    }
-  }
-
-  public long ping() {
-    this.lastPing = System.currentTimeMillis();
-    return lastPing;
-  }
-
-  public abstract void onStart();
-
-  public abstract void onTerminate();
-
-  public final void start() {
-    Logger logger = LoggerFactory.getLogger(getClass());
-    logger.debug("Process[{}] starting", name);
-    if (this.port != null) {
-      lastPing = System.currentTimeMillis();
-      pingTask = monitor.scheduleWithFixedDelay(breakOnMissingPing, 5, 5, TimeUnit.SECONDS);
-    }
-    this.onStart();
-    logger.debug("Process[{}] started", name);
-  }
-
-  public final void terminate() {
-    Logger logger = LoggerFactory.getLogger(getClass());
-    logger.debug("Process[{}] terminating", name);
-    if (monitor != null) {
-      this.monitor.shutdownNow();
-      this.monitor = null;
-      if (this.pingTask != null) {
-        this.pingTask.cancel(true);
-        this.pingTask = null;
-      }
-      this.onTerminate();
-    }
-    logger.debug("Process[{}] terminated", name);
-  }
-}

server/sonar-process/src/main/java/org/sonar/process/ProcessMXBean.java

@@ -19,15 +19,9 @@
  */
 package org.sonar.process;
 
-public interface ProcessMXBean {
-
-  String IS_READY = "isReady";
-  String PING = "ping";
-  String TERMINATE = "terminate";
+public interface ProcessMXBean extends Terminatable {
 
   boolean isReady();
 
   long ping();
-
-  void terminate();
 }

server/sonar-process/src/main/java/org/sonar/process/ProcessUtils.java

@@ -0,0 +1,70 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.process;
+
+import org.apache.commons.io.IOUtils;
+
+import javax.annotation.Nullable;
+
+public class ProcessUtils {
+  private ProcessUtils() {
+    // only static stuff
+  }
+
+  public static boolean isAlive(@Nullable Process process) {
+    if (process == null) {
+      return false;
+    }
+    try {
+      process.exitValue();
+      return false;
+    } catch (IllegalThreadStateException e) {
+      return true;
+    }
+  }
+
+  public static void destroyQuietly(@Nullable Process process) {
+    if (process != null && isAlive(process)) {
+      try {
+        process.destroy();
+      } catch (Exception ignored) {
+        // ignored
+      }
+    }
+  }
+
+  public static void addSelfShutdownHook(final Terminatable terminatable) {
+    Thread shutdownHook = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        terminatable.terminate();
+      }
+    });
+    Runtime.getRuntime().addShutdownHook(shutdownHook);
+  }
+
+  public static void closeStreams(@Nullable Process process) {
+    if (process != null) {
+      IOUtils.closeQuietly(process.getInputStream());
+      IOUtils.closeQuietly(process.getOutputStream());
+      IOUtils.closeQuietly(process.getErrorStream());
+    }
+  }
+}

server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java

@@ -25,12 +25,14 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
 import javax.management.JMX;
 import javax.management.MBeanServerConnection;
 import javax.management.remote.JMXConnector;
 import javax.management.remote.JMXConnectorFactory;
 import javax.management.remote.JMXServiceURL;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -48,13 +50,18 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Fork and monitor a new process
  */
-public class ProcessWrapper extends Thread {
+public class ProcessWrapper extends Thread implements Terminatable {
 
   private final static Logger LOGGER = LoggerFactory.getLogger(ProcessWrapper.class);
+  public static final long READY_TIMEOUT_MS = 120000L;
 
   private String processName, className;
   private int jmxPort = -1;
@@ -64,7 +71,7 @@ public class ProcessWrapper extends Thread {
   private final Properties properties = new Properties();
   private File workDir;
   private File propertiesFile;
-  private java.lang.Process process;
+  private Process process;
   private StreamGobbler errorGobbler;
   private StreamGobbler outputGobbler;
   private ProcessMXBean processMXBean;
@@ -115,7 +122,16 @@ public class ProcessWrapper extends Thread {
     return this;
   }
 
-  public ProcessWrapper execute() {
+  @CheckForNull
+  Process process() {
+    return process;
+  }
+
+  /**
+   * Execute command-line and connects to JMX RMI.
+   * @return true on success, false if bad command-line or process failed to start JMX RMI
+   */
+  public boolean execute() {
     List<String> command = new ArrayList<String>();
     command.add(buildJavaCommand());
     command.addAll(javaOpts);
@@ -137,7 +153,11 @@ public class ProcessWrapper extends Thread {
       outputGobbler.start();
       errorGobbler.start();
       processMXBean = waitForJMX();
-      return this;
+      if (processMXBean == null) {
+        terminate();
+        return false;
+      }
+      return true;
     } catch (IOException e) {
       throw new IllegalStateException("Fail to start command: " + StringUtils.join(command, " "), e);
     }
@@ -146,13 +166,15 @@ public class ProcessWrapper extends Thread {
   @Override
   public void run() {
     try {
-      process.waitFor();
-    } catch (InterruptedException e) {
+      if (ProcessUtils.isAlive(process)) {
+        process.waitFor();
+      }
+    } catch (Exception e) {
       LOGGER.info("ProcessThread has been interrupted. Killing process.");
     } finally {
       waitUntilFinish(outputGobbler);
       waitUntilFinish(errorGobbler);
-      closeStreams(process);
+      ProcessUtils.closeStreams(process);
       FileUtils.deleteQuietly(propertiesFile);
       processMXBean = null;
     }
@@ -177,14 +199,6 @@ public class ProcessWrapper extends Thread {
     }
   }
 
-  private void closeStreams(@Nullable java.lang.Process process) {
-    if (process != null) {
-      IOUtils.closeQuietly(process.getInputStream());
-      IOUtils.closeQuietly(process.getOutputStream());
-      IOUtils.closeQuietly(process.getErrorStream());
-    }
-  }
-
   private String buildJavaCommand() {
     String separator = System.getProperty("file.separator");
     return System.getProperty("java.home")
@@ -211,8 +225,7 @@ public class ProcessWrapper extends Thread {
       propertiesFile = File.createTempFile("sq-conf", "properties");
       Properties props = new Properties();
       props.putAll(properties);
-      props.put(Process.NAME_PROPERTY, processName);
-      props.put(Process.PORT_PROPERTY, String.valueOf(jmxPort));
+      props.put(MonitoredProcess.NAME_PROPERTY, processName);
       OutputStream out = new FileOutputStream(propertiesFile);
       props.store(out, "Temporary properties file for Process [" + getName() + "]");
       out.close();
@@ -222,52 +235,84 @@ public class ProcessWrapper extends Thread {
     }
   }
 
-  private ProcessMXBean waitForJMX() {
-    Exception exception = null;
+  /**
+   * Wait for JMX RMI to be ready. Return <code>null</code>
+   */
+  @CheckForNull
+  private ProcessMXBean waitForJMX() throws UnknownHostException, MalformedURLException {
+    String path = "/jndi/rmi://" + InetAddress.getLocalHost().getHostName() + ":" + jmxPort + "/jmxrmi";
+    JMXServiceURL jmxUrl = new JMXServiceURL("rmi", InetAddress.getLocalHost().getHostAddress(), jmxPort, path);
+
     for (int i = 0; i < 5; i++) {
       try {
-        Thread.sleep(1000);
-      } catch (InterruptedException e) {
-        throw new IllegalStateException("Could not connect to JMX server", e);
-      }
-      LOGGER.debug("Try #{} to connect to JMX server for process '{}'", i, processName);
-      try {
-        String protocol = "rmi";
-        String path = "/jndi/rmi://" + InetAddress.getLocalHost().getHostName() + ":" + jmxPort + "/jmxrmi";
-        JMXServiceURL jmxUrl = new JMXServiceURL(protocol, InetAddress.getLocalHost().getHostAddress(), jmxPort, path);
+        Thread.sleep(1000L);
+        LOGGER.debug("Try #{} to connect to JMX server for process '{}'", i, processName);
         JMXConnector jmxConnector = JMXConnectorFactory.connect(jmxUrl, null);
         MBeanServerConnection mBeanServer = jmxConnector.getMBeanServerConnection();
-        ProcessMXBean bean = JMX.newMBeanProxy(mBeanServer, Process.objectNameFor(processName), ProcessMXBean.class);
-        LOGGER.info("{} process up and running, listening to its state with url: '{}'", getName(), jmxUrl.toString());
+        ProcessMXBean bean = JMX.newMBeanProxy(mBeanServer, JmxUtils.objectName(processName), ProcessMXBean.class);
         return bean;
-      } catch (MalformedURLException e) {
-        throw new IllegalStateException("JMXUrl is not valid", e);
-      } catch (UnknownHostException e) {
-        throw new IllegalStateException("Could not get hostname", e);
-      } catch (IOException e) {
-        exception = e;
+      } catch (Exception ignored) {
+        // ignored
       }
     }
-    throw new IllegalStateException("Could not connect to JMX service", exception);
+    // failed to connect
+    return null;
   }
 
+  @Override
   public void terminate() {
     if (processMXBean != null) {
-      LOGGER.info("Stopping {} process", getName());
-      processMXBean.terminate();
+      // Send the terminate command to process in order to gracefully shutdown.
+      // Then hardly kill it if it didn't terminate in 30 seconds
+      ScheduledExecutorService killer = Executors.newScheduledThreadPool(1);
       try {
-        this.join();
-      } catch (InterruptedException e) {
+        Runnable killerTask = new Runnable() {
+          @Override
+          public void run() {
+            ProcessUtils.destroyQuietly(process);
+          }
+        };
+
+        ScheduledFuture killerFuture = killer.schedule(killerTask, 30, TimeUnit.SECONDS);
+        LOGGER.info("Stopping {} process", getName());
+        processMXBean.terminate();
+        killerFuture.cancel(true);
+        processMXBean = null;
+        LOGGER.info("{} process stopped", getName());
+
+      } catch (Exception e) {
+        LOGGER.warn("Failed to terminate " + getName(), e);
+      } finally {
+        killer.shutdownNow();
+      }
+    } else {
+      // process is not monitored through JMX, but killing it though
+      ProcessUtils.destroyQuietly(process);
+    }
+  }
+
+  public boolean waitForReady() throws InterruptedException {
+    if (processMXBean == null) {
+      return false;
+    }
+    long now = 0;
+    long wait = 500L;
+    while (now < READY_TIMEOUT_MS) {
+      try {
+        if (processMXBean.isReady()) {
+          return true;
+        }
+      } catch (Exception e) {
         // ignore
       }
-      processMXBean = null;
-      LOGGER.info("{} process stopped", getName());
+      Thread.sleep(wait);
+      now += wait;
     }
+    return false;
   }
 
   private static class StreamGobbler extends Thread {
     private final InputStream is;
-    private volatile Exception exception;
     private final String pName;
 
     StreamGobbler(InputStream is, String name) {
@@ -285,17 +330,13 @@ public class ProcessWrapper extends Thread {
         while ((line = br.readLine()) != null) {
           LOGGER.info(pName + " > " + line);
         }
-      } catch (IOException ioe) {
-        exception = ioe;
+      } catch (IOException ignored) {
+        // ignored
 
       } finally {
         IOUtils.closeQuietly(br);
         IOUtils.closeQuietly(isr);
       }
     }
-
-    public Exception getException() {
-      return exception;
-    }
   }
 }

server/sonar-process/src/main/java/org/sonar/process/Terminatable.java

@@ -0,0 +1,30 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.process;
+
+/**
+ * This interface was not named Stopable in order to not conflict with {@link Thread#stop()}.
+ */
+public interface Terminatable {
+  /**
+   * Stops pending work. Must <b>not</b> throw an exception on error.
+   */
+  void terminate();
+}

server/sonar-process/src/test/java/org/sonar/process/ProcessTest.java

@@ -1,179 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.process;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import javax.management.JMX;
-import javax.management.MBeanServer;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.net.ServerSocket;
-import java.util.Properties;
-
-import static junit.framework.TestCase.fail;
-import static org.fest.assertions.Assertions.assertThat;
-
-public class ProcessTest {
-
-  int freePort;
-
-  @Before
-  public void setup() throws IOException {
-    ServerSocket socket = new ServerSocket(0);
-    freePort = socket.getLocalPort();
-    socket.close();
-  }
-
-  Process process;
-
-  @After
-  public void tearDown() throws Exception {
-    if (process != null) {
-      MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
-      mbeanServer.unregisterMBean(process.getObjectName());
-    }
-  }
-
-  @Test
-  public void fails_invalid_name() {
-    try {
-      Process.objectNameFor("::");
-      fail();
-    } catch (Exception e) {
-      assertThat(e.getMessage()).isEqualTo("Cannot create ObjectName for ::");
-    }
-  }
-
-  @Test
-  public void fail_missing_properties() {
-    Properties properties = new Properties();
-    try {
-      new TestProcess(new Props(properties));
-    } catch (Exception e) {
-      assertThat(e.getMessage()).isEqualTo(Process.MISSING_NAME_ARGUMENT);
-    }
-  }
-
-  @Test
-  public void should_register_mbean() throws Exception {
-
-    MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
-
-    Properties properties = new Properties();
-    properties.setProperty(Process.NAME_PROPERTY, "TEST");
-    Props props = new Props(properties);
-    process = new TestProcess(props);
-
-    // 0 Can have a valid ObjectName
-    assertThat(process.getObjectName()).isNotNull();
-
-    // 1 assert that process MBean is registered
-    assertThat(mbeanServer.isRegistered(process.getObjectName())).isTrue();
-
-    // 2 assert that we cannot make another Process in the same JVM
-    try {
-      process = new TestProcess(props);
-      fail();
-    } catch (IllegalStateException e) {
-      assertThat(e.getMessage()).isEqualTo("Process already exists in current JVM");
-    }
-  }
-
-  @Test(timeout = 5000L)
-  @Ignore
-  public void should_stop_explicit() throws Exception {
-    Properties properties = new Properties();
-    properties.setProperty(Process.NAME_PROPERTY, "TEST");
-    Props props = new Props(properties);
-    process = new TestProcess(props);
-
-    MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
-    final ProcessMXBean processMXBean = JMX.newMBeanProxy(mbeanServer, process.getObjectName(), ProcessMXBean.class);
-
-    Thread procThread = new Thread(new Runnable() {
-      @Override
-      public void run() {
-        process.start();
-      }
-    });
-
-    // 0. Process is loaded but not ready yet.
-    assertThat(processMXBean.isReady()).isFalse();
-
-    // 1. Pretend the process has started
-    procThread.start();
-    Thread.sleep(200);
-    assertThat(procThread.isAlive()).isTrue();
-    assertThat(processMXBean.isReady()).isTrue();
-
-    // 2. Stop the process through Management
-    processMXBean.terminate();
-    procThread.join();
-  }
-
-  @Test(timeout = 15000L)
-  @Ignore
-  public void should_stop_implicit() throws Exception {
-    Properties properties = new Properties();
-    properties.setProperty(Process.NAME_PROPERTY, "TEST");
-    properties.setProperty(Process.PORT_PROPERTY, Integer.toString(freePort));
-    Props props = new Props(properties);
-    process = new TestProcess(props);
-
-    process.start();
-  }
-
-  public static class TestProcess extends Process {
-
-    private boolean ready = false;
-    private boolean running = false;
-
-    public TestProcess(Props props) {
-      super(props);
-      running = true;
-    }
-
-    @Override
-    public void onStart() {
-      ready = true;
-      while (running) {
-        try {
-          Thread.sleep(1000);
-        } catch (InterruptedException e) {
-          e.printStackTrace();
-        }
-      }
-    }
-
-    @Override
-    public void onTerminate() {
-      running = false;
-    }
-
-    @Override
-    public boolean isReady() {
-      return ready;
-    }
-  }
-}

server/sonar-search/src/main/java/org/sonar/search/ElasticSearch.java → server/sonar-search/src/main/java/org/sonar/search/SearchServer.java

@@ -26,13 +26,13 @@ import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeBuilder;
 import org.slf4j.LoggerFactory;
 import org.sonar.process.ConfigurationUtils;
-import org.sonar.process.Process;
+import org.sonar.process.MonitoredProcess;
 import org.sonar.process.Props;
 import org.sonar.search.script.ListUpdate;
 
 import java.io.File;
 
-public class ElasticSearch extends Process {
+public class SearchServer extends MonitoredProcess {
 
   public static final String ES_DEBUG_PROPERTY = "esDebug";
   public static final String ES_PORT_PROPERTY = "sonar.search.port";
@@ -40,84 +40,26 @@ public class ElasticSearch extends Process {
 
   private Node node;
 
-  ElasticSearch(Props props) {
+  SearchServer(Props props) throws Exception {
     super(props);
   }
 
   @Override
-  public boolean isReady() {
-    try {
-      return (node.client().admin().cluster().prepareHealth()
-        .setWaitForYellowStatus()
-        .setTimeout(TimeValue.timeValueSeconds(3L))
-        .get()
-        .getStatus() != ClusterHealthStatus.RED);
-    } catch (Exception e) {
-      return false;
-    }
-  }
-
-  private void initAnalysis(ImmutableSettings.Builder esSettings) {
-    esSettings
-      .put("index.mapper.dynamic", false)
-
-        // Sortable text analyzer
-      .put("index.analysis.analyzer.sortable.type", "custom")
-      .put("index.analysis.analyzer.sortable.tokenizer", "keyword")
-      .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate")
-
-        // Edge NGram index-analyzer
-      .put("index.analysis.analyzer.index_grams.type", "custom")
-      .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace")
-      .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter")
-
-        // Edge NGram search-analyzer
-      .put("index.analysis.analyzer.search_grams.type", "custom")
-      .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace")
-      .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase")
-
-        // Word index-analyzer
-      .put("index.analysis.analyzer.index_words.type", "custom")
-      .put("index.analysis.analyzer.index_words.tokenizer", "standard")
-      .putArray("index.analysis.analyzer.index_words.filter",
-        "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem")
-
-        // Word search-analyzer
-      .put("index.analysis.analyzer.search_words.type", "custom")
-      .put("index.analysis.analyzer.search_words.tokenizer", "standard")
-      .putArray("index.analysis.analyzer.search_words.filter",
-        "standard", "lowercase", "stop", "asciifolding", "porter_stem")
-
-        // Edge NGram filter
-      .put("index.analysis.filter.gram_filter.type", "edgeNGram")
-      .put("index.analysis.filter.gram_filter.min_gram", 2)
-      .put("index.analysis.filter.gram_filter.max_gram", 15)
-      .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol")
-
-        // Word filter
-      .put("index.analysis.filter.word_filter.type", "word_delimiter")
-      .put("index.analysis.filter.word_filter.generate_word_parts", true)
-      .put("index.analysis.filter.word_filter.catenate_words", true)
-      .put("index.analysis.filter.word_filter.catenate_numbers", true)
-      .put("index.analysis.filter.word_filter.catenate_all", true)
-      .put("index.analysis.filter.word_filter.split_on_case_change", true)
-      .put("index.analysis.filter.word_filter.preserve_original", true)
-      .put("index.analysis.filter.word_filter.split_on_numerics", true)
-      .put("index.analysis.filter.word_filter.stem_english_possessive", true)
-
-        // Path Analyzer
-      .put("index.analysis.analyzer.path_analyzer.type", "custom")
-      .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy");
-
+  protected boolean doIsReady() {
+    return (node.client().admin().cluster().prepareHealth()
+      .setWaitForYellowStatus()
+      .setTimeout(TimeValue.timeValueSeconds(3L))
+      .get()
+      .getStatus() != ClusterHealthStatus.RED);
   }
 
   @Override
-  public void onStart() {
+  protected void doStart() {
     String dataDir = props.of("sonar.path.data");
     Integer port = props.intOf(ES_PORT_PROPERTY);
     String clusterName = props.of(ES_CLUSTER_PROPERTY);
 
-    LoggerFactory.getLogger(ElasticSearch.class).info("Starting ES[{}] on port: {}", clusterName, port);
+    LoggerFactory.getLogger(SearchServer.class).info("Starting ES[{}] on port: {}", clusterName, port);
 
     ImmutableSettings.Builder esSettings = ImmutableSettings.settingsBuilder()
       .put("es.foreground", "yes")
@@ -165,15 +107,70 @@ public class ElasticSearch extends Process {
     }
   }
 
-  public void onTerminate() {
+  private void initAnalysis(ImmutableSettings.Builder esSettings) {
+    esSettings
+      .put("index.mapper.dynamic", false)
+
+      // Sortable text analyzer
+      .put("index.analysis.analyzer.sortable.type", "custom")
+      .put("index.analysis.analyzer.sortable.tokenizer", "keyword")
+      .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate")
+
+      // Edge NGram index-analyzer
+      .put("index.analysis.analyzer.index_grams.type", "custom")
+      .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace")
+      .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter")
+
+      // Edge NGram search-analyzer
+      .put("index.analysis.analyzer.search_grams.type", "custom")
+      .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace")
+      .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase")
+
+      // Word index-analyzer
+      .put("index.analysis.analyzer.index_words.type", "custom")
+      .put("index.analysis.analyzer.index_words.tokenizer", "standard")
+      .putArray("index.analysis.analyzer.index_words.filter",
+        "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem")
+
+      // Word search-analyzer
+      .put("index.analysis.analyzer.search_words.type", "custom")
+      .put("index.analysis.analyzer.search_words.tokenizer", "standard")
+      .putArray("index.analysis.analyzer.search_words.filter",
+        "standard", "lowercase", "stop", "asciifolding", "porter_stem")
+
+      // Edge NGram filter
+      .put("index.analysis.filter.gram_filter.type", "edgeNGram")
+      .put("index.analysis.filter.gram_filter.min_gram", 2)
+      .put("index.analysis.filter.gram_filter.max_gram", 15)
+      .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol")
+
+      // Word filter
+      .put("index.analysis.filter.word_filter.type", "word_delimiter")
+      .put("index.analysis.filter.word_filter.generate_word_parts", true)
+      .put("index.analysis.filter.word_filter.catenate_words", true)
+      .put("index.analysis.filter.word_filter.catenate_numbers", true)
+      .put("index.analysis.filter.word_filter.catenate_all", true)
+      .put("index.analysis.filter.word_filter.split_on_case_change", true)
+      .put("index.analysis.filter.word_filter.preserve_original", true)
+      .put("index.analysis.filter.word_filter.split_on_numerics", true)
+      .put("index.analysis.filter.word_filter.stem_english_possessive", true)
+
+      // Path Analyzer
+      .put("index.analysis.analyzer.path_analyzer.type", "custom")
+      .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy");
+
+  }
+
+  @Override
+  protected void doTerminate() {
     if (node != null && !node.isClosed()) {
       node.close();
       node = null;
     }
   }
 
-  public static void main(String... args) throws InterruptedException {
+  public static void main(String... args) throws Exception {
     Props props = ConfigurationUtils.loadPropsFromCommandLineArgs(args);
-    new ElasticSearch(props).start();
+    new SearchServer(props).start();
   }
 }

server/sonar-search/src/main/resources/logback.xml

@@ -27,21 +27,9 @@
     </encoder>
   </appender>
 
-  <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
-    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
-      <level>WARN</level>
-    </filter>
-    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
-      <pattern>
-        %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n
-      </pattern>
-    </encoder>
-  </appender>
-
   <root>
     <level value="INFO"/>
     <appender-ref ref="LOGFILE"/>
-    <appender-ref ref="CONSOLE"/>
   </root>
 
 </configuration>

server/sonar-search/src/test/java/org/sonar/search/ElasticSearchTest.java → server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java

@@ -28,26 +28,27 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
-import org.sonar.process.Process;
+import org.sonar.process.JmxUtils;
+import org.sonar.process.MonitoredProcess;
 import org.sonar.process.Props;
 
 import javax.management.InstanceNotFoundException;
 import javax.management.MBeanRegistrationException;
 import javax.management.MBeanServer;
+
 import java.io.File;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.net.ServerSocket;
-import java.net.SocketException;
 import java.util.Properties;
 
 import static org.fest.assertions.Assertions.assertThat;
 import static org.junit.Assert.fail;
 
-public class ElasticSearchTest {
+public class SearchServerTest {
 
   File tempDirectory;
-  ElasticSearch elasticSearch;
+  SearchServer searchServer;
   int freePort;
   int freeESPort;
 
@@ -64,7 +65,6 @@ public class ElasticSearchTest {
     socket.close();
   }
 
-
   @After
   public void tearDown() throws MBeanRegistrationException, InstanceNotFoundException {
     resetMBeanServer();
@@ -73,32 +73,32 @@ public class ElasticSearchTest {
   private void resetMBeanServer() throws MBeanRegistrationException, InstanceNotFoundException {
     try {
       MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
-      mbeanServer.unregisterMBean(Process.objectNameFor("ES"));
+      mbeanServer.unregisterMBean(JmxUtils.objectName("ES"));
     } catch (Exception e) {
       e.printStackTrace();
     }
   }
 
   @Test
-  public void can_connect() throws SocketException {
+  public void can_connect() throws Exception {
     Properties properties = new Properties();
-    properties.setProperty(Process.NAME_PROPERTY, "ES");
+    properties.setProperty(MonitoredProcess.NAME_PROPERTY, "ES");
     properties.setProperty("sonar.path.data", tempDirectory.getAbsolutePath());
     properties.setProperty("sonar.path.logs", tempDirectory.getAbsolutePath());
-    properties.setProperty(ElasticSearch.ES_PORT_PROPERTY, Integer.toString(freeESPort));
-    properties.setProperty(ElasticSearch.ES_CLUSTER_PROPERTY, "sonarqube");
+    properties.setProperty(SearchServer.ES_PORT_PROPERTY, Integer.toString(freeESPort));
+    properties.setProperty(SearchServer.ES_CLUSTER_PROPERTY, "sonarqube");
 
-    elasticSearch = new ElasticSearch(new Props(properties));
+    searchServer = new SearchServer(new Props(properties));
     new Thread(new Runnable() {
       @Override
       public void run() {
-        elasticSearch.start();
+        searchServer.start();
       }
     }).start();
-    assertThat(elasticSearch.isReady()).isFalse();
+    assertThat(searchServer.isReady()).isFalse();
 
     int count = 0;
-    while (!elasticSearch.isReady() && count < 100) {
+    while (!searchServer.isReady() && count < 100) {
       try {
         Thread.sleep(500);
       } catch (InterruptedException e) {
@@ -114,13 +114,11 @@ public class ElasticSearchTest {
     TransportClient client = new TransportClient(settings)
       .addTransportAddress(new InetSocketTransportAddress("localhost", freeESPort));
 
-
     // 0 assert that we have a OK cluster available
     assertThat(client.admin().cluster().prepareClusterStats().get().getStatus()).isEqualTo(ClusterHealthStatus.GREEN);
 
-
     // 2 assert that we can shut down ES
-    elasticSearch.terminate();
+    searchServer.terminate();
     try {
       client.admin().cluster().prepareClusterStats().get().getStatus();
       fail();

server/sonar-server/src/main/java/org/sonar/server/app/EmbeddedTomcat.java

@@ -19,15 +19,17 @@
  */
 package org.sonar.server.app;
 
-import org.apache.catalina.LifecycleException;
 import org.apache.catalina.connector.Connector;
 import org.apache.catalina.startup.Tomcat;
 import org.apache.commons.io.FileUtils;
+import org.slf4j.LoggerFactory;
+import org.sonar.process.ProcessUtils;
 import org.sonar.process.Props;
+import org.sonar.process.Terminatable;
 
 import java.io.File;
 
-class EmbeddedTomcat {
+class EmbeddedTomcat implements Terminatable {
 
   private final Props props;
   private Tomcat tomcat = null;
@@ -64,60 +66,22 @@ class EmbeddedTomcat {
       Logging.configure(tomcat, props);
       Connectors.configure(tomcat, props);
       Webapp.configure(tomcat, props);
+      ProcessUtils.addSelfShutdownHook(this);
       tomcat.start();
-      addShutdownHook();
       ready = true;
       tomcat.getServer().await();
     } catch (Exception e) {
       throw new IllegalStateException("Fail to start web server", e);
+    } finally {
+      // Failed to start or received a shutdown command (should never occur as shutdown port is disabled)
+      terminate();
     }
-    stop();
   }
 
   private File tomcatBasedir() {
     return new File(props.of("sonar.path.temp"), "tomcat");
   }
 
-  private void addShutdownHook() {
-    hook = new Thread() {
-      @Override
-      public void run() {
-        EmbeddedTomcat.this.doStop();
-      }
-    };
-    Runtime.getRuntime().addShutdownHook(hook);
-  }
-
-
-  void stop() {
-    removeShutdownHook();
-    doStop();
-  }
-
-  private synchronized void doStop() {
-    try {
-      if (tomcat != null && !stopping) {
-        stopping = true;
-        tomcat.stop();
-        tomcat.destroy();
-      }
-      tomcat = null;
-      stopping = false;
-      ready = false;
-      FileUtils.deleteQuietly(tomcatBasedir());
-
-    } catch (LifecycleException e) {
-      throw new IllegalStateException("Fail to stop web server", e);
-    }
-  }
-
-  private void removeShutdownHook() {
-    if (hook != null && !hook.isAlive()) {
-      Runtime.getRuntime().removeShutdownHook(hook);
-      hook = null;
-    }
-  }
-
   boolean isReady() {
     return ready && tomcat != null;
   }
@@ -129,4 +93,21 @@ class EmbeddedTomcat {
     }
     return -1;
   }
+
+  @Override
+  public void terminate() {
+    if (tomcat != null && !stopping) {
+      try {
+        stopping = true;
+        tomcat.stop();
+        tomcat.destroy();
+      } catch (Exception e) {
+        LoggerFactory.getLogger(EmbeddedTomcat.class).error("Fail to stop web service", e);
+      }
+    }
+    tomcat = null;
+    stopping = false;
+    ready = false;
+    FileUtils.deleteQuietly(tomcatBasedir());
+  }
 }

server/sonar-server/src/main/java/org/sonar/server/app/ServerProcess.java → server/sonar-server/src/main/java/org/sonar/server/app/WebServer.java

@@ -21,19 +21,20 @@ package org.sonar.server.app;
 
 import org.slf4j.LoggerFactory;
 import org.sonar.process.ConfigurationUtils;
+import org.sonar.process.MonitoredProcess;
 import org.sonar.process.Props;
 
-public class ServerProcess extends org.sonar.process.Process {
+public class WebServer extends MonitoredProcess {
 
   private final EmbeddedTomcat tomcat;
 
-  ServerProcess(Props props) {
+  WebServer(Props props) throws Exception {
     super(props);
     this.tomcat = new EmbeddedTomcat(props);
   }
 
   @Override
-  public void onStart() {
+  protected void doStart() {
     try {
       tomcat.start();
     } catch (Exception e) {
@@ -44,18 +45,18 @@ public class ServerProcess extends org.sonar.process.Process {
   }
 
   @Override
-  public void onTerminate() {
-    tomcat.stop();
+  protected void doTerminate() {
+    tomcat.terminate();
   }
 
   @Override
-  public boolean isReady() {
+  protected boolean doIsReady() {
     return tomcat.isReady();
   }
 
-  public static void main(String[] args) {
+  public static void main(String[] args) throws Exception {
     Props props = ConfigurationUtils.loadPropsFromCommandLineArgs(args);
     Logging.init(props);
-    new ServerProcess(props).start();
+    new WebServer(props).start();
   }
 }

sonar-application/src/main/java/org/sonar/application/App.java

@@ -21,124 +21,77 @@ package org.sonar.application;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.sonar.process.JmxUtils;
 import org.sonar.process.Monitor;
-import org.sonar.process.Process;
 import org.sonar.process.ProcessMXBean;
+import org.sonar.process.ProcessUtils;
 import org.sonar.process.ProcessWrapper;
 
-import javax.management.InstanceAlreadyExistsException;
-import javax.management.MBeanRegistrationException;
-import javax.management.MBeanServer;
-import javax.management.NotCompliantMBeanException;
-import java.lang.management.ManagementFactory;
-
 public class App implements ProcessMXBean {
 
-  static final String PROCESS_NAME = "SonarQube";
-
   static final String SONAR_WEB_PROCESS = "web";
   static final String SONAR_SEARCH_PROCESS = "search";
 
-
   private final Installation installation;
-  private Monitor monitor;
+
+  private final Monitor monitor = new Monitor();
   private ProcessWrapper elasticsearch;
   private ProcessWrapper server;
 
   public App(Installation installation) throws Exception {
     this.installation = installation;
-
-    Thread shutdownHook = new Thread(new Runnable() {
-      @Override
-      public void run() {
-        terminate();
-      }
-    });
-    Runtime.getRuntime().addShutdownHook(shutdownHook);
-
-    MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
-    try {
-      mbeanServer.registerMBean(this, Process.objectNameFor(PROCESS_NAME));
-    } catch (InstanceAlreadyExistsException e) {
-      throw new IllegalStateException("Process already exists in current JVM", e);
-    } catch (MBeanRegistrationException e) {
-      throw new IllegalStateException("Could not register process as MBean", e);
-    } catch (NotCompliantMBeanException e) {
-      throw new IllegalStateException("Process is not a compliant MBean", e);
-    }
-
-    monitor = new Monitor();
+    JmxUtils.registerMBean(this, "SonarQube");
+    ProcessUtils.addSelfShutdownHook(this);
   }
 
-  public void start() {
-
-    Logger logger = LoggerFactory.getLogger(getClass());
-
-    logger.info("Starting search server");
-    elasticsearch = new ProcessWrapper(SONAR_SEARCH_PROCESS)
-      .setWorkDir(installation.homeDir())
-      .setJmxPort(Integer.parseInt(installation.prop(DefaultSettings.ES_JMX_PORT_KEY)))
-      .addJavaOpts(installation.prop(DefaultSettings.ES_JAVA_OPTS_KEY))
-      .addJavaOpts(String.format("-Djava.io.tmpdir=%s", installation.tempDir().getAbsolutePath()))
-      .addJavaOpts(String.format("-D%s=%s", DefaultSettings.PATH_LOGS_KEY, installation.logsDir().getAbsolutePath()))
-      .setClassName("org.sonar.search.ElasticSearch")
-      .setProperties(installation.props().encryptedProperties())
-      .addClasspath(installation.starPath("lib/common"))
-      .addClasspath(installation.starPath("lib/search"))
-      .execute();
-    monitor.registerProcess(elasticsearch);
-    logger.info("Search server is ready");
-
-    logger.info("Starting web server");
-    server = new ProcessWrapper(SONAR_WEB_PROCESS)
-      .setWorkDir(installation.homeDir())
-      .setJmxPort(Integer.parseInt(installation.prop(DefaultSettings.WEB_JMX_PORT_KEY)))
-      .addJavaOpts(installation.prop(DefaultSettings.WEB_JAVA_OPTS_KEY))
-      .addJavaOpts(DefaultSettings.WEB_JAVA_OPTS_APPENDED_VAL)
-      .addJavaOpts(String.format("-Djava.io.tmpdir=%s", installation.tempDir().getAbsolutePath()))
-      .addJavaOpts(String.format("-D%s=%s", DefaultSettings.PATH_LOGS_KEY, installation.logsDir().getAbsolutePath()))
-      .setClassName("org.sonar.server.app.ServerProcess")
-      .setProperties(installation.props().encryptedProperties())
-      .addClasspath(installation.starPath("extensions/jdbc-driver/mysql"))
-      .addClasspath(installation.starPath("extensions/jdbc-driver/mssql"))
-      .addClasspath(installation.starPath("extensions/jdbc-driver/oracle"))
-      .addClasspath(installation.starPath("extensions/jdbc-driver/postgresql"))
-      .addClasspath(installation.starPath("lib/common"))
-      .addClasspath(installation.starPath("lib/server"))
-      .execute();
-    monitor.registerProcess(server);
-    logger.info("Web server is ready");
-
-    monitor.start();
-
+  public void start() throws InterruptedException {
     try {
-      monitor.join();
-    } catch (InterruptedException e) {
-      // TODO ignore ?
-
+      Logger logger = LoggerFactory.getLogger(getClass());
+      monitor.start();
+
+      elasticsearch = new ProcessWrapper(SONAR_SEARCH_PROCESS)
+        .setWorkDir(installation.homeDir())
+        .setJmxPort(Integer.parseInt(installation.prop(DefaultSettings.ES_JMX_PORT_KEY)))
+        .addJavaOpts(installation.prop(DefaultSettings.ES_JAVA_OPTS_KEY))
+        .addJavaOpts(String.format("-Djava.io.tmpdir=%s", installation.tempDir().getAbsolutePath()))
+        .addJavaOpts(String.format("-Dsonar.path.logs=%s", installation.logsDir().getAbsolutePath()))
+        .setClassName("org.sonar.search.SearchServer")
+        .setProperties(installation.props().encryptedProperties())
+        .addClasspath(installation.starPath("lib/common"))
+        .addClasspath(installation.starPath("lib/search"));
+      if (elasticsearch.execute()) {
+        monitor.registerProcess(elasticsearch);
+        if (elasticsearch.waitForReady()) {
+          logger.info("Search server is ready");
+
+          server = new ProcessWrapper(SONAR_WEB_PROCESS)
+            .setWorkDir(installation.homeDir())
+            .setJmxPort(Integer.parseInt(installation.prop(DefaultSettings.WEB_JMX_PORT_KEY)))
+            .addJavaOpts(installation.prop(DefaultSettings.WEB_JAVA_OPTS_KEY))
+            .addJavaOpts(DefaultSettings.WEB_JAVA_OPTS_APPENDED_VAL)
+            .addJavaOpts(String.format("-Djava.io.tmpdir=%s", installation.tempDir().getAbsolutePath()))
+            .setClassName("org.sonar.server.app.WebServer")
+            .setProperties(installation.props().encryptedProperties())
+            .addClasspath(installation.starPath("extensions/jdbc-driver/mysql"))
+            .addClasspath(installation.starPath("extensions/jdbc-driver/mssql"))
+            .addClasspath(installation.starPath("extensions/jdbc-driver/oracle"))
+            .addClasspath(installation.starPath("extensions/jdbc-driver/postgresql"))
+            .addClasspath(installation.starPath("lib/common"))
+            .addClasspath(installation.starPath("lib/server"));
+          if (server.execute()) {
+            monitor.registerProcess(server);
+            if (server.waitForReady()) {
+              logger.info("Web server is ready");
+              monitor.join();
+            }
+          }
+        }
+      }
     } finally {
-      logger.debug("Closing App because monitor is gone.");
       terminate();
     }
   }
 
-  @Override
-  public void terminate() {
-    Logger logger = LoggerFactory.getLogger(getClass());
-
-    if (monitor != null) {
-      monitor.interrupt();
-      monitor = null;
-      if (elasticsearch != null) {
-        elasticsearch.terminate();
-      }
-      if (server != null) {
-        server.terminate();
-      }
-      logger.info("Stopping SonarQube main process");
-    }
-  }
-
   @Override
   public boolean isReady() {
     return monitor.isAlive();
@@ -149,6 +102,18 @@ public class App implements ProcessMXBean {
     return System.currentTimeMillis();
   }
 
+  @Override
+  public void terminate() {
+    monitor.terminate();
+    monitor.interrupt();
+    if (server != null) {
+      server.terminate();
+    }
+    if (elasticsearch != null) {
+      elasticsearch.terminate();
+    }
+  }
+
   public static void main(String[] args) throws Exception {
     Installation installation = new Installation();
     new AppLogging().configure(installation);

sonar-application/src/test/java/org/sonar/application/AppTest.java

@@ -22,22 +22,14 @@ package org.sonar.application;
 import org.apache.commons.io.FileUtils;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Rule;
-import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.process.Process;
-import org.sonar.process.ProcessMXBean;
 
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
 import java.io.File;
-import java.lang.management.ManagementFactory;
 
 import static org.fest.assertions.Assertions.assertThat;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 
 public class AppTest {