From 3d598f644c2afeb7b8bb34edc8ba6f53e523ee2e Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Wed, 30 Jul 2014 16:06:30 +0200 Subject: [PATCH] SONAR-4898 fix handling of server early termination --- .../main/java/org/sonar/process/JmxUtils.java | 3 ++ .../main/java/org/sonar/process/Monitor.java | 21 +++++++++---- .../org/sonar/process/ProcessWrapper.java | 9 ++++-- .../org/sonar/server/app/EmbeddedTomcat.java | 31 +++++++++++-------- .../java/org/sonar/server/app/Webapp.java | 3 +- .../PlatformServletContextListener.java | 3 +- .../plugins/ServerPluginJarsInstaller.java | 17 +++++----- .../main/java/org/sonar/application/App.java | 14 ++++----- 8 files changed, 61 insertions(+), 40 deletions(-) diff --git a/server/sonar-process/src/main/java/org/sonar/process/JmxUtils.java b/server/sonar-process/src/main/java/org/sonar/process/JmxUtils.java index b1c28932118..b88973e45c9 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/JmxUtils.java +++ b/server/sonar-process/src/main/java/org/sonar/process/JmxUtils.java @@ -30,6 +30,9 @@ public class JmxUtils { // only static stuff } + public static final String WEB_SERVER_NAME = "web"; + public static final String SEARCH_SERVER_NAME = "search"; + public static ObjectName objectName(String name) { try { return new ObjectName("org.sonar", "name", name); diff --git a/server/sonar-process/src/main/java/org/sonar/process/Monitor.java b/server/sonar-process/src/main/java/org/sonar/process/Monitor.java index 2d6054ee11a..a695286f9c8 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/Monitor.java +++ b/server/sonar-process/src/main/java/org/sonar/process/Monitor.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -38,7 +39,7 @@ public class Monitor extends Thread implements Terminable { private final static Logger LOGGER = LoggerFactory.getLogger(Monitor.class); private volatile List processes; - private volatile Map pings; + private final Map pings; private final ScheduledFuture watch; private final ScheduledExecutorService monitor; @@ -48,7 +49,7 @@ public class Monitor extends Thread implements Terminable { public Monitor() { super("Process Monitor"); processes = new ArrayList(); - pings = new HashMap(); + pings = new ConcurrentHashMap(); monitor = Executors.newScheduledThreadPool(1); watch = monitor.scheduleWithFixedDelay(new ProcessWatch(), 0L, PING_DELAY_MS, TimeUnit.MILLISECONDS); } @@ -99,14 +100,17 @@ public class Monitor extends Thread implements Terminable { @Override public void run() { try { - while (true) { + boolean ok = true; + while (ok) { for (ProcessWrapper process : processes) { if (!processIsValid(process)) { - LOGGER.warn("Monitor::run() -- Process '{}' is not valid. Exiting monitor", process.getName()); + ok = false; interrupt(); } } - Thread.sleep(PING_DELAY_MS); + if (ok) { + Thread.sleep(PING_DELAY_MS); + } } } catch (InterruptedException e) { LOGGER.debug("Monitoring thread is interrupted"); @@ -117,12 +121,17 @@ public class Monitor extends Thread implements Terminable { @Override public void terminate() { - processes.clear(); if (!monitor.isShutdown()) { monitor.shutdownNow(); } if (!watch.isCancelled()) { watch.cancel(true); } + + for (int i=processes.size()-1 ; i>=0 ; i--) { + processes.get(i).terminate(); + } + processes.clear(); + interrupt(); } } diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java index f5773a294b1..9529a799898 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java +++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java @@ -176,9 +176,7 @@ public class ProcessWrapper extends Thread implements Terminable { waitUntilFinish(errorGobbler); ProcessUtils.closeStreams(process); FileUtils.deleteQuietly(propertiesFile); - processMXBean = null; } - LOGGER.trace("ProcessWrapper::run() END"); } public boolean isReady() { @@ -261,7 +259,7 @@ public class ProcessWrapper extends Thread implements Terminable { @Override public void terminate() { - if (processMXBean != null) { + if (processMXBean != null && process != null) { // Send the terminate command to process in order to gracefully shutdown. // Then hardly kill it if it didn't terminate in 30 seconds ScheduledExecutorService killer = Executors.newScheduledThreadPool(1); @@ -280,6 +278,7 @@ public class ProcessWrapper extends Thread implements Terminable { } catch (Exception ignored) { // ignore + ignored.printStackTrace(); } finally { killer.shutdownNow(); } @@ -287,6 +286,7 @@ public class ProcessWrapper extends Thread implements Terminable { // process is not monitored through JMX, but killing it though ProcessUtils.destroyQuietly(process); } + processMXBean = null; } public boolean waitForReady() throws InterruptedException { @@ -297,6 +297,9 @@ public class ProcessWrapper extends Thread implements Terminable { long wait = 500L; while (now < READY_TIMEOUT_MS) { try { + if (processMXBean == null) { + return false; + } if (processMXBean.isReady()) { return true; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/app/EmbeddedTomcat.java b/server/sonar-server/src/main/java/org/sonar/server/app/EmbeddedTomcat.java index ef6541dbc2f..4553b5867cf 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/app/EmbeddedTomcat.java +++ b/server/sonar-server/src/main/java/org/sonar/server/app/EmbeddedTomcat.java @@ -20,6 +20,7 @@ package org.sonar.server.app; import org.apache.catalina.connector.Connector; +import org.apache.catalina.core.StandardContext; import org.apache.catalina.startup.Tomcat; import org.apache.commons.io.FileUtils; import org.slf4j.LoggerFactory; @@ -34,7 +35,7 @@ class EmbeddedTomcat implements Terminable { private final Props props; private Tomcat tomcat = null; private Thread hook = null; - private boolean stopping = false, ready = false; + private boolean ready = false; EmbeddedTomcat(Props props) { this.props = props; @@ -65,11 +66,14 @@ class EmbeddedTomcat implements Terminable { Logging.configure(tomcat, props); Connectors.configure(tomcat, props); - Webapp.configure(tomcat, props); + StandardContext webappContext = Webapp.configure(tomcat, props); ProcessUtils.addSelfShutdownHook(this); tomcat.start(); - ready = true; - tomcat.getServer().await(); + + if (webappContext.getState().isAvailable()) { + ready = true; + tomcat.getServer().await(); + } } catch (Exception e) { throw new IllegalStateException("Fail to start web server", e); } finally { @@ -96,17 +100,18 @@ class EmbeddedTomcat implements Terminable { @Override public void terminate() { - if (tomcat != null && !stopping) { - try { - stopping = true; - tomcat.stop(); - tomcat.destroy(); - } catch (Exception e) { - LoggerFactory.getLogger(EmbeddedTomcat.class).error("Fail to stop web service", e); + if (tomcat != null) { + synchronized (tomcat) { + if (tomcat.getServer().getState().isAvailable()) { + try { + tomcat.stop(); + tomcat.destroy(); + } catch (Exception e) { + LoggerFactory.getLogger(EmbeddedTomcat.class).error("Fail to stop web service", e); + } + } } } - tomcat = null; - stopping = false; ready = false; FileUtils.deleteQuietly(tomcatBasedir()); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/app/Webapp.java b/server/sonar-server/src/main/java/org/sonar/server/app/Webapp.java index 986b35487c7..2371978637e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/app/Webapp.java +++ b/server/sonar-server/src/main/java/org/sonar/server/app/Webapp.java @@ -33,7 +33,7 @@ class Webapp { private static final String RAILS_ENV = "rails.env"; private static final String PROPERTY_CONTEXT = "sonar.web.context"; - static void configure(Tomcat tomcat, Props props) { + static StandardContext configure(Tomcat tomcat, Props props) { try { String webDir = props.of("sonar.path.web"); LoggerFactory.getLogger(Webapp.class).info("Webapp directory: " + webDir); @@ -55,6 +55,7 @@ class Webapp { } configureRailsMode(props, context); context.setJarScanner(new NullJarScanner()); + return context; } catch (Exception e) { throw new IllegalStateException("Fail to configure webapp", e); diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/PlatformServletContextListener.java b/server/sonar-server/src/main/java/org/sonar/server/platform/PlatformServletContextListener.java index c6b7d2f6346..adb4f401cd2 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/PlatformServletContextListener.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/PlatformServletContextListener.java @@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory; import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; + import java.util.Enumeration; import java.util.Properties; @@ -49,7 +50,7 @@ public final class PlatformServletContextListener implements ServletContextListe // unexpected errors LoggerFactory.getLogger(getClass()).error("Fail to start server", t); stopQuietly(); - throw new IllegalStateException("Fail to start server", t); + throw new IllegalStateException("Fail to start webapp", t); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/plugins/ServerPluginJarsInstaller.java b/server/sonar-server/src/main/java/org/sonar/server/plugins/ServerPluginJarsInstaller.java index 3884439e166..d9ea45f1b44 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/plugins/ServerPluginJarsInstaller.java +++ b/server/sonar-server/src/main/java/org/sonar/server/plugins/ServerPluginJarsInstaller.java @@ -20,7 +20,6 @@ package org.sonar.server.plugins; import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.io.FileUtils; @@ -53,7 +52,7 @@ public class ServerPluginJarsInstaller { private final ServerUpgradeStatus serverUpgradeStatus; public ServerPluginJarsInstaller(Server server, ServerUpgradeStatus serverUpgradeStatus, - DefaultServerFileSystem fs, ServerPluginJarInstaller installer) { + DefaultServerFileSystem fs, ServerPluginJarInstaller installer) { this.server = server; this.serverUpgradeStatus = serverUpgradeStatus; this.fs = fs; @@ -97,7 +96,7 @@ public class ServerPluginJarsInstaller { private void moveDownloadedPlugins() { if (fs.getDownloadedPluginsDir().exists()) { - Collection sourceFiles = FileUtils.listFiles(fs.getDownloadedPluginsDir(), new String[]{"jar"}, false); + Collection sourceFiles = FileUtils.listFiles(fs.getDownloadedPluginsDir(), new String[] {"jar"}, false); for (File sourceFile : sourceFiles) { overridePlugin(sourceFile, true); } @@ -117,7 +116,6 @@ public class ServerPluginJarsInstaller { } } - private void overridePlugin(File sourceFile, boolean deleteSource) { File destDir = fs.getUserPluginsDir(); File destFile = new File(destDir, sourceFile.getName()); @@ -181,7 +179,7 @@ public class ServerPluginJarsInstaller { public List getUninstalls() { List names = Lists.newArrayList(); if (fs.getTrashPluginsDir().exists()) { - List files = (List) FileUtils.listFiles(fs.getTrashPluginsDir(), new String[]{"jar"}, false); + List files = (List) FileUtils.listFiles(fs.getTrashPluginsDir(), new String[] {"jar"}, false); for (File file : files) { names.add(file.getName()); } @@ -191,7 +189,7 @@ public class ServerPluginJarsInstaller { public void cancelUninstalls() { if (fs.getTrashPluginsDir().exists()) { - List files = (List) FileUtils.listFiles(fs.getTrashPluginsDir(), new String[]{"jar"}, false); + List files = (List) FileUtils.listFiles(fs.getTrashPluginsDir(), new String[] {"jar"}, false); for (File file : files) { try { FileUtils.moveFileToDirectory(file, fs.getUserPluginsDir(), false); @@ -211,9 +209,10 @@ public class ServerPluginJarsInstaller { private void deploy(DefaultPluginMetadata plugin) { LOG.info("Deploy plugin {}", Joiner.on(" / ").skipNulls().join(plugin.getName(), plugin.getVersion(), plugin.getImplementationBuild())); - Preconditions.checkState(plugin.isCompatibleWith(server.getVersion()), - "Plugin %s needs a more recent version of SonarQube than %s. At least %s is expected", - plugin.getKey(), server.getVersion(), plugin.getSonarVersion()); + if (!plugin.isCompatibleWith(server.getVersion())) { + throw MessageException.of(String.format("Plugin %s needs a more recent version of SonarQube than %s. At least %s is expected", + plugin.getKey(), server.getVersion(), plugin.getSonarVersion())); + } try { File pluginDeployDir = new File(fs.getDeployedPluginsDir(), plugin.getKey()); diff --git a/sonar-application/src/main/java/org/sonar/application/App.java b/sonar-application/src/main/java/org/sonar/application/App.java index a45bfd2fa0c..5d105718b19 100644 --- a/sonar-application/src/main/java/org/sonar/application/App.java +++ b/sonar-application/src/main/java/org/sonar/application/App.java @@ -29,9 +29,6 @@ import org.sonar.process.ProcessWrapper; public class App implements ProcessMXBean { - static final String SONAR_WEB_PROCESS = "web"; - static final String SONAR_SEARCH_PROCESS = "search"; - private final Installation installation; private final Monitor monitor = new Monitor(); @@ -49,7 +46,7 @@ public class App implements ProcessMXBean { Logger logger = LoggerFactory.getLogger(getClass()); monitor.start(); - elasticsearch = new ProcessWrapper(SONAR_SEARCH_PROCESS) + elasticsearch = new ProcessWrapper(JmxUtils.SEARCH_SERVER_NAME) .setWorkDir(installation.homeDir()) .setJmxPort(Integer.parseInt(installation.prop(DefaultSettings.ES_JMX_PORT_KEY))) .addJavaOpts(installation.prop(DefaultSettings.ES_JAVA_OPTS_KEY)) @@ -64,7 +61,7 @@ public class App implements ProcessMXBean { if (elasticsearch.waitForReady()) { logger.info("Search server is ready"); - server = new ProcessWrapper(SONAR_WEB_PROCESS) + server = new ProcessWrapper(JmxUtils.WEB_SERVER_NAME) .setWorkDir(installation.homeDir()) .setJmxPort(Integer.parseInt(installation.prop(DefaultSettings.WEB_JMX_PORT_KEY))) .addJavaOpts(installation.prop(DefaultSettings.WEB_JAVA_OPTS_KEY)) @@ -105,8 +102,11 @@ public class App implements ProcessMXBean { @Override public void terminate() { - monitor.terminate(); - monitor.interrupt(); + LoggerFactory.getLogger(App.class).info("Stopping"); + if (monitor.isAlive()) { + monitor.terminate(); + monitor.interrupt(); + } if (server != null) { server.terminate(); } -- 2.39.5