SONAR-4898 - Fixed gracefull shutdown with JSW

7 files changed, 51 insertions, 54 deletions

diff --git a/server/sonar-process/src/main/java/org/sonar/process/Monitor.java b/server/sonar-process/src/main/java/org/sonar/process/Monitor.java
index 0a218a3d349..ba70dd71b27 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/Monitor.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/Monitor.java
@@ -33,6 +33,8 @@ import java.util.concurrent.TimeUnit;
 
 public class Monitor extends Thread {
 
+  private static final long MAX_TIME = 15000L;
+
   private final static Logger LOGGER = LoggerFactory.getLogger(Monitor.class);
 
   private volatile List<ProcessWrapper> processes;
@@ -52,8 +54,8 @@ public class Monitor extends Thread {
     processes.add(processWrapper);
     pings.put(processWrapper.getName(), System.currentTimeMillis());
     processWrapper.start();
-    for(int i=0; i<10; i++){
-      if(processWrapper.getProcessMXBean() == null || !processWrapper.getProcessMXBean().isReady()){
+    for (int i = 0; i < 10; i++) {
+      if (processWrapper.getProcessMXBean() == null || !processWrapper.getProcessMXBean().isReady()) {
         try {
           Thread.sleep(500L);
         } catch (InterruptedException e) {
@@ -82,30 +84,30 @@ public class Monitor extends Thread {
     long now = System.currentTimeMillis();
     LOGGER.debug("Monitor::processIsValid() -- Time since last ping for '{}': {}ms",
       process.getName(), (now - pings.get(process.getName())));
-    return (now - pings.get(process.getName())) < 5000L;
+    return (now - pings.get(process.getName())) < MAX_TIME;
   }
 
   public void run() {
-    LOGGER.trace("Monitor::run() START");
+    LOGGER.debug("Monitor::run() START");
     boolean everythingOK = true;
-    while (everythingOK) {
-      for(ProcessWrapper process: processes){
-        if(!processIsValid(process)){
-          LOGGER.warn("Monitor::run() -- Process '{}' is not valid. Exiting monitor", process.getName());
-          everythingOK = false;
-          break;
+    try {
+      while (everythingOK) {
+        for (ProcessWrapper process : processes) {
+          if (!processIsValid(process)) {
+            LOGGER.warn("Monitor::run() -- Process '{}' is not valid. Exiting monitor", process.getName());
+            everythingOK = false;
+            break;
+          }
         }
+        Thread.sleep(3000L);
       }
-      try {
-        Thread.sleep(1000);
-      } catch (InterruptedException e) {
-        LOGGER.warn("Monitoring thread has been interrupted. Closing");
-        watch.cancel(true);
-        monitor.shutdownNow();
-      }
+
+    } catch (InterruptedException e) {
+      LOGGER.info("Monitoring thread is interrupted.");
+    } finally {
+      watch.cancel(true);
+      monitor.shutdownNow();
     }
-    watch.cancel(true);
-    monitor.shutdownNow();
-    LOGGER.trace("Monitor::run() END");
+    LOGGER.debug("Monitor::run() END");
   }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/Process.java b/server/sonar-process/src/main/java/org/sonar/process/Process.java
index cc064b758b7..d7fef28593d 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/Process.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/Process.java
@@ -46,7 +46,7 @@ public abstract class Process implements ProcessMXBean {
   public static final String PORT_PROPERTY = "pPort";
   public static final String MISSING_NAME_ARGUMENT = "Missing Name argument";
 
-  private final static Logger LOGGER = LoggerFactory.getLogger(Process.class);
+  protected final static Logger LOGGER = LoggerFactory.getLogger(Process.class);
 
   private Long lastPing;
 
@@ -56,9 +56,9 @@ public abstract class Process implements ProcessMXBean {
   protected final Props props;
   private Thread shutdownHook;
 
-  private static final long MAX_ALLOWED_TIME = 3000L;
+  private static final long MAX_ALLOWED_TIME = 15000L;
   private ScheduledFuture<?> pingTask = null;
-  final ScheduledExecutorService monitor = Executors.newScheduledThreadPool(1);
+  private ScheduledExecutorService monitor = Executors.newScheduledThreadPool(1);
   final Runnable breakOnMissingPing = new Runnable() {
     public void run() {
       long time = System.currentTimeMillis();
@@ -123,13 +123,7 @@ public abstract class Process implements ProcessMXBean {
     shutdownHook = new Thread(new Runnable() {
       @Override
       public void run() {
-        LOGGER.trace("Process[{}]::ShutdownHook::run() START", name);
-        Process.this.onTerminate();
-        if (Process.this.pingTask != null) {
-          Process.this.pingTask.cancel(true);
-        }
-        Process.this.monitor.shutdownNow();
-        LOGGER.trace("Process[{}]::ShutdownHook::run() END", name);
+        terminate();
       }
     });
     Runtime.getRuntime().addShutdownHook(shutdownHook);
@@ -166,22 +160,17 @@ public abstract class Process implements ProcessMXBean {
     LOGGER.trace("Process[{}]::start() END", name);
   }
 
-  public final void terminate(boolean waitForTermination) {
-    LOGGER.trace("Process[{}]::terminate() START", name);
-    Runtime.getRuntime().removeShutdownHook(shutdownHook);
-    shutdownHook.start();
-    if (waitForTermination) {
-      try {
-        shutdownHook.join();
-      } catch (InterruptedException e) {
-        System.exit(-1);
+  public final void terminate() {
+    LOGGER.debug("Process[{}]::terminate() START", name);
+    if (monitor != null) {
+      this.monitor.shutdownNow();
+      this.monitor = null;
+      if (this.pingTask != null) {
+        this.pingTask.cancel(true);
+        this.pingTask = null;
       }
+      this.onTerminate();
     }
-    LOGGER.trace("Process[{}]::terminate() END", name);
-    System.exit(1);
-  }
-
-  public final void terminate() {
-    terminate(false);
+    LOGGER.debug("Process[{}]::terminate() END", name);
   }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java
index cb1f77c11f6..04fa01d26d2 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java
@@ -69,6 +69,7 @@ public class ProcessWrapper extends Thread {
   private ProcessMXBean processMXBean;
 
   public ProcessWrapper(String processName) {
+    super(processName);
     this.processName = processName;
   }
 
@@ -151,6 +152,7 @@ public class ProcessWrapper extends Thread {
     } catch (InterruptedException e) {
       e.printStackTrace();
     } finally {
+      LOGGER.info("ProcessThread has been interrupted. Killing process.");
       process.destroy();
       waitUntilFinish(outputGobbler);
       waitUntilFinish(errorGobbler);
@@ -173,7 +175,7 @@ public class ProcessWrapper extends Thread {
       try {
         thread.join();
       } catch (InterruptedException e) {
-        LOGGER.error("InterruptedException while waiting finish of " + thread.toString(), e);
+        LOGGER.error("InterruptedException while waiting finish of " + thread.getName() + " in process '" + getName() + "'", e);
       }
     }
   }
@@ -255,7 +257,12 @@ public class ProcessWrapper extends Thread {
   public void terminate() {
     if (processMXBean != null) {
       processMXBean.terminate();
-      waitUntilFinish(this);
+      try {
+        Thread.sleep(10000L);
+      } catch (InterruptedException e) {
+        LOGGER.warn("Could bnit", e);
+      }
+      this.interrupt();
     }
   }
 
diff --git a/server/sonar-search/src/main/java/org/sonar/search/ElasticSearch.java b/server/sonar-search/src/main/java/org/sonar/search/ElasticSearch.java
index ea1b4f95b15..dc1aae83833 100644
--- a/server/sonar-search/src/main/java/org/sonar/search/ElasticSearch.java
+++ b/server/sonar-search/src/main/java/org/sonar/search/ElasticSearch.java
@@ -164,23 +164,21 @@ public class ElasticSearch extends Process {
       try {
         Thread.sleep(100);
       } catch (InterruptedException e) {
-        node.close();
+        ;
       }
     }
-    terminate();
   }
 
   public void onTerminate() {
     if (node != null && !node.isClosed()) {
       node.close();
-    }
-    if (node != null) {
-      node.stop();
+      node = null;
     }
   }
 
   public static void main(String... args) throws InterruptedException {
     final ElasticSearch elasticSearch = new ElasticSearch(args);
     elasticSearch.start();
+    LOGGER.info("ElasticSearch is done.");
   }
 }
diff --git a/server/sonar-search/src/main/resources/logback.xml b/server/sonar-search/src/main/resources/logback.xml
index 25d1932da9d..1588ae0641d 100644
--- a/server/sonar-search/src/main/resources/logback.xml
+++ b/server/sonar-search/src/main/resources/logback.xml
@@ -45,5 +45,5 @@
     <appender-ref ref="LOGFILE"/>
     <appender-ref ref="CONSOLE"/>
   </root>
-
+  <logger name="org.sonar.process" level="debug"/>
 </configuration>
diff --git a/server/sonar-search/src/test/java/org/sonar/search/ElasticSearchTest.java b/server/sonar-search/src/test/java/org/sonar/search/ElasticSearchTest.java
index e8185c549a4..62950bee734 100644
--- a/server/sonar-search/src/test/java/org/sonar/search/ElasticSearchTest.java
+++ b/server/sonar-search/src/test/java/org/sonar/search/ElasticSearchTest.java
@@ -118,7 +118,7 @@ public class ElasticSearchTest {
 
 
     // 2 assert that we can shut down ES
-    elasticSearch.terminate(true);
+    elasticSearch.terminate();
     try {
       client.admin().cluster().prepareClusterStats().get().getStatus();
       fail();
diff --git a/server/sonar-server-app/src/main/java/org/sonar/server/app/ServerProcess.java b/server/sonar-server-app/src/main/java/org/sonar/server/app/ServerProcess.java
index 9006705365e..ee303b8313b 100644
--- a/server/sonar-server-app/src/main/java/org/sonar/server/app/ServerProcess.java
+++ b/server/sonar-server-app/src/main/java/org/sonar/server/app/ServerProcess.java
@@ -47,5 +47,6 @@ public class ServerProcess extends org.sonar.process.Process {
 
   public static void main(String[] args) {
     new ServerProcess(args).start();
+    LOGGER.info("ServerProcess is done");
   }
 }