SONAR-12043 main process supports graceful and hard stop

10 files changed, 255 insertions, 65 deletions

diff --git a/server/sonar-process/build.gradle b/server/sonar-process/build.gradle
index d458ec793f0..ee525939cb9 100644
--- a/server/sonar-process/build.gradle
+++ b/server/sonar-process/build.gradle
@@ -27,5 +27,6 @@ dependencies {
   testCompile 'org.assertj:assertj-core'
   testCompile 'org.eclipse.jetty:jetty-server'
   testCompile 'org.mockito:mockito-core'
+  testCompile 'org.awaitility:awaitility'
   testCompile project(':sonar-testing-harness')
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/HardStopWatcher.java b/server/sonar-process/src/main/java/org/sonar/process/AbstractStopWatcher.java
index 7286d64259a..50e4e73940a 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/HardStopWatcher.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/AbstractStopWatcher.java
@@ -19,36 +19,32 @@
  */
 package org.sonar.process;
 
-import org.slf4j.LoggerFactory;
-import org.sonar.process.sharedmemoryfile.ProcessCommands;
+import com.google.common.annotations.VisibleForTesting;
+import java.util.function.BooleanSupplier;
 
-/**
- * This watchdog is looking for hard stop to be requested via {@link ProcessCommands#askedForHardStop()}.
- */
-public class HardStopWatcher extends Thread {
-
-  private final ProcessCommands commands;
-  private final Stoppable stoppable;
+abstract class AbstractStopWatcher extends Thread {
+  private final Runnable stopCommand;
+  private final BooleanSupplier shouldStopTest;
   private final long delayMs;
-  private boolean watching = true;
+  private volatile boolean watching = true;
 
-  public HardStopWatcher(ProcessCommands commands, Stoppable stoppable) {
-    this(commands, stoppable, 500L);
+  public AbstractStopWatcher(String threadName, Runnable stopCommand, BooleanSupplier shouldStopTest) {
+    this(threadName, stopCommand, shouldStopTest, 500L);
   }
 
-  HardStopWatcher(ProcessCommands commands, Stoppable stoppable, long delayMs) {
-    super("HardStop Watcher");
-    this.commands = commands;
-    this.stoppable = stoppable;
+  @VisibleForTesting
+  AbstractStopWatcher(String threadName, Runnable stopCommand, BooleanSupplier shouldStopTest, long delayMs) {
+    super(threadName);
+    this.stopCommand = stopCommand;
+    this.shouldStopTest = shouldStopTest;
     this.delayMs = delayMs;
   }
 
   @Override
   public void run() {
     while (watching) {
-      if (commands.askedForHardStop()) {
-        LoggerFactory.getLogger(getClass()).info("Hard stopping process");
-        stoppable.hardStopAsync();
+      if (shouldStopTest.getAsBoolean()) {
+        stopCommand.run();
         watching = false;
       } else {
         try {
@@ -63,6 +59,7 @@ public class HardStopWatcher extends Thread {
   }
 
   public void stopWatching() {
+    super.interrupt();
     watching = false;
   }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/HardStopperThread.java b/server/sonar-process/src/main/java/org/sonar/process/AbstractStopperThread.java
index c2414dd17ba..ae0c85d0018 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/HardStopperThread.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/AbstractStopperThread.java
@@ -28,14 +28,15 @@ import org.slf4j.LoggerFactory;
 /**
  * Stops process in a short time fashion
  */
-class HardStopperThread extends Thread {
+abstract class AbstractStopperThread extends Thread {
 
-  private final Monitored monitored;
+  private final Runnable stopCode;
   private final long terminationTimeoutMs;
+  private boolean stop = false;
 
-  HardStopperThread(Monitored monitored, long terminationTimeoutMs) {
-    super("HardStopper");
-    this.monitored = monitored;
+  AbstractStopperThread(String threadName, Runnable stopCode, long terminationTimeoutMs) {
+    super(threadName);
+    this.stopCode = stopCode;
     this.terminationTimeoutMs = terminationTimeoutMs;
   }
 
@@ -43,11 +44,18 @@ class HardStopperThread extends Thread {
   public void run() {
     ExecutorService executor = Executors.newSingleThreadExecutor();
     try {
-      Future future = executor.submit(monitored::hardStop);
+      Future future = executor.submit(stopCode);
       future.get(terminationTimeoutMs, TimeUnit.MILLISECONDS);
     } catch (Exception e) {
-      LoggerFactory.getLogger(getClass()).error("Can not stop in {}ms", terminationTimeoutMs, e);
+      if (!stop) {
+        LoggerFactory.getLogger(getClass()).error("Can not stop in {}ms", terminationTimeoutMs, e);
+      }
     }
     executor.shutdownNow();
   }
+
+  public void stopIt() {
+    this.stop = true;
+    super.interrupt();
+  }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/Monitored.java b/server/sonar-process/src/main/java/org/sonar/process/Monitored.java
index 56d6b249ce0..653f5408220 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/Monitored.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/Monitored.java
@@ -47,6 +47,11 @@ public interface Monitored {
   void awaitStop();
 
   /**
+   * Ask process to gracefully stop and wait until then.
+   */
+  void stop();
+
+  /**
    * Ask process to quickly stop and wait until then.
    */
   void hardStop();
diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java
index ae6b4376557..1976f5699b4 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java
@@ -20,12 +20,17 @@
 package org.sonar.process;
 
 import java.io.File;
+import java.util.Optional;
+import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.process.sharedmemoryfile.DefaultProcessCommands;
 import org.sonar.process.sharedmemoryfile.ProcessCommands;
 
-public class ProcessEntryPoint implements Stoppable {
+import static java.util.Optional.of;
+import static java.util.Optional.ofNullable;
+
+public class ProcessEntryPoint {
 
   public static final String PROPERTY_PROCESS_KEY = "process.key";
   public static final String PROPERTY_PROCESS_INDEX = "process.index";
@@ -39,18 +44,19 @@ public class ProcessEntryPoint implements Stoppable {
   private final Lifecycle lifecycle = new Lifecycle();
   private final ProcessCommands commands;
   private final SystemExit exit;
+  private final StopWatcher stopWatcher;
   private final HardStopWatcher hardStopWatcher;
-  private volatile Monitored monitored;
-  private volatile HardStopperThread hardStopperThread;
-
   // new Runnable() is important to avoid conflict of call to ProcessEntryPoint#stop() with Thread#stop()
-  private Thread shutdownHook = new Thread(new Runnable() {
+  private final Thread shutdownHook = new Thread(new Runnable() {
     @Override
     public void run() {
       exit.setInShutdownHook();
-      hardStop();
+      stop();
     }
   });
+  private volatile Monitored monitored;
+  private volatile StopperThread stopperThread;
+  private volatile HardStopperThread hardStopperThread;
 
   ProcessEntryPoint(Props props, SystemExit exit, ProcessCommands commands) {
     this(props, getProcessNumber(props), getSharedDir(props), exit, commands);
@@ -63,6 +69,7 @@ public class ProcessEntryPoint implements Stoppable {
     this.sharedDir = sharedDir;
     this.exit = exit;
     this.commands = commands;
+    this.stopWatcher = new StopWatcher(commands, this);
     this.hardStopWatcher = new HardStopWatcher(commands, this);
   }
 
@@ -108,6 +115,7 @@ public class ProcessEntryPoint implements Stoppable {
   private void launch(Logger logger) throws InterruptedException {
     logger.info("Starting {}", getKey());
     Runtime.getRuntime().addShutdownHook(shutdownHook);
+    stopWatcher.start();
     hardStopWatcher.start();
 
     monitored.start();
@@ -152,31 +160,63 @@ public class ProcessEntryPoint implements Stoppable {
     return lifecycle.getState() == Lifecycle.State.STARTED;
   }
 
+  void stop() {
+    stopAsync()
+      .ifPresent(stoppingThread -> {
+        try {
+          // join() does nothing if thread already finished
+          stoppingThread.join();
+          lifecycle.tryToMoveTo(Lifecycle.State.STOPPED);
+          commands.endWatch();
+          exit.exit(0);
+        } catch (InterruptedException e) {
+          // stop can be aborted by a hard stop
+          Thread.currentThread().interrupt();
+        }
+      });
+  }
+
+  private Optional<StopperThread> stopAsync() {
+    if (lifecycle.tryToMoveTo(Lifecycle.State.STOPPING)) {
+      long terminationTimeoutMs = Long.parseLong(props.nonNullValue(PROPERTY_TERMINATION_TIMEOUT_MS));
+      stopperThread = new StopperThread(monitored, terminationTimeoutMs);
+      stopperThread.start();
+      stopWatcher.stopWatching();
+      return of(stopperThread);
+    }
+    // stopperThread could already exist
+    return ofNullable(stopperThread);
+  }
+
   /**
    * Blocks until stopped in a timely fashion (see {@link HardStopperThread})
    */
   void hardStop() {
-    hardStopAsync();
-    try {
-      // hardStopperThread is not null for sure
-      // join() does nothing if thread already finished
-      hardStopperThread.join();
-      lifecycle.tryToMoveTo(Lifecycle.State.STOPPED);
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-    }
-    commands.endWatch();
-    exit.exit(0);
+    hardStopAsync()
+      .ifPresent(stoppingThread -> {
+        try {
+          // join() does nothing if thread already finished
+          stoppingThread.join();
+          lifecycle.tryToMoveTo(Lifecycle.State.STOPPED);
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+        }
+        commands.endWatch();
+        exit.exit(0);
+      });
   }
 
-  @Override
-  public void hardStopAsync() {
+  private Optional<HardStopperThread> hardStopAsync() {
     if (lifecycle.tryToMoveTo(Lifecycle.State.HARD_STOPPING)) {
       long terminationTimeoutMs = Long.parseLong(props.nonNullValue(PROPERTY_TERMINATION_TIMEOUT_MS));
-      hardStopperThread = new HardStopperThread(monitored, terminationTimeoutMs);
+      hardStopperThread = new HardStopperThread(monitored, terminationTimeoutMs, stopperThread);
       hardStopperThread.start();
       hardStopWatcher.stopWatching();
+      stopWatcher.stopWatching();
+      return of(hardStopperThread);
     }
+    // hardStopperThread could already exist
+    return ofNullable(hardStopperThread);
   }
 
   Lifecycle.State getState() {
@@ -202,4 +242,68 @@ public class ProcessEntryPoint implements Stoppable {
   private static File getSharedDir(Props props) {
     return props.nonNullValueAsFile(PROPERTY_SHARED_PATH);
   }
+
+  /**
+   * This watchdog is looking for hard stop to be requested via {@link ProcessCommands#askedForHardStop()}.
+   */
+  private static class HardStopWatcher extends AbstractStopWatcher {
+
+    private HardStopWatcher(ProcessCommands commands, ProcessEntryPoint processEntryPoint) {
+      super(
+        "HardStop Watcher",
+        () -> {
+          LoggerFactory.getLogger(HardStopWatcher.class).info("Hard stopping process");
+          processEntryPoint.hardStopAsync();
+        },
+        commands::askedForHardStop);
+    }
+
+  }
+
+  /**
+   * This watchdog is looking for graceful stop to be requested via {@link ProcessCommands#askedForStop()} ()}.
+   */
+  private static class StopWatcher extends AbstractStopWatcher {
+
+    private StopWatcher(ProcessCommands commands, ProcessEntryPoint processEntryPoint) {
+      super(
+        "Stop Watcher",
+        () -> {
+          LoggerFactory.getLogger(StopWatcher.class).info("Stopping process");
+          processEntryPoint.stopAsync();
+        },
+        commands::askedForStop);
+    }
+
+  }
+
+  /**
+   * Stops process in a graceful fashion
+   */
+  private static class StopperThread extends AbstractStopperThread {
+
+    private StopperThread(Monitored monitored, long terminationTimeoutMs) {
+      super("Stopper", monitored::stop, terminationTimeoutMs);
+    }
+
+  }
+
+  /**
+   * Stops process in a short time fashion
+   */
+  private static class HardStopperThread extends AbstractStopperThread {
+
+    private HardStopperThread(Monitored monitored, long terminationTimeoutMs, @Nullable StopperThread stopperThread) {
+      super(
+        "HardStopper",
+        () -> {
+          if (stopperThread != null) {
+            stopperThread.stopIt();
+          }
+          monitored.hardStop();
+        },
+        terminationTimeoutMs);
+    }
+
+  }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/Stoppable.java b/server/sonar-process/src/main/java/org/sonar/process/Stoppable.java
index 7e189583aef..cadc0ba58b2 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/Stoppable.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/Stoppable.java
@@ -21,6 +21,8 @@ package org.sonar.process;
 
 public interface Stoppable {
 
+  void stopAsync();
+
   void hardStopAsync();
 
 }
diff --git a/server/sonar-process/src/test/java/org/sonar/process/HardStopWatcherTest.java b/server/sonar-process/src/test/java/org/sonar/process/AbstractStopWatcherTest.java
index c8806d05f21..8fbc9b2cefd 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/HardStopWatcherTest.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/AbstractStopWatcherTest.java
@@ -26,29 +26,33 @@ import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
 import org.sonar.process.sharedmemoryfile.ProcessCommands;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-public class HardStopWatcherTest {
+public class AbstractStopWatcherTest {
 
   @Rule
   public TestRule safeguardTimeout = new DisableOnDebug(Timeout.seconds(60));
 
   @Test
   public void stop_if_receive_command() throws Exception {
+
     ProcessCommands commands = mock(ProcessCommands.class);
     when(commands.askedForHardStop()).thenReturn(false, true);
     Stoppable stoppable = mock(Stoppable.class);
 
-    HardStopWatcher underTest = new HardStopWatcher(commands, stoppable, 1L);
+    AbstractStopWatcher underTest = new AbstractStopWatcher("TheThreadName",
+      stoppable::hardStopAsync, commands::askedForHardStop, 1L) {};
     underTest.start();
 
     while (underTest.isAlive()) {
       Thread.sleep(1L);
     }
     verify(stoppable).hardStopAsync();
+    assertThat(underTest.getName()).isEqualTo("TheThreadName");
   }
 
   @Test
@@ -57,7 +61,8 @@ public class HardStopWatcherTest {
     when(commands.askedForHardStop()).thenReturn(false);
     Stoppable stoppable = mock(Stoppable.class);
 
-    HardStopWatcher underTest = new HardStopWatcher(commands, stoppable, 1L);
+    AbstractStopWatcher underTest = new AbstractStopWatcher("TheThreadName",
+      stoppable::hardStopAsync, commands::askedForHardStop, 1L) {};
     underTest.start();
     underTest.interrupt();
 
@@ -65,5 +70,6 @@ public class HardStopWatcherTest {
       Thread.sleep(1L);
     }
     verify(stoppable, never()).hardStopAsync();
+    assertThat(underTest.getName()).isEqualTo("TheThreadName");
   }
 }
diff --git a/server/sonar-process/src/test/java/org/sonar/process/HardStopperThreadTest.java b/server/sonar-process/src/test/java/org/sonar/process/AbstractStopperThreadTest.java
index a53b8f8a23c..a3e4050a14d 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/HardStopperThreadTest.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/AbstractStopperThreadTest.java
@@ -19,42 +19,60 @@
  */
 package org.sonar.process;
 
+import java.util.concurrent.TimeUnit;
 import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.timeout;
 import static org.mockito.Mockito.verify;
 
-public class HardStopperThreadTest {
+public class AbstractStopperThreadTest {
   private Monitored monitored = mock(Monitored.class);
 
   @Test
   public void stop_in_a_timely_fashion() {
     // max stop timeout is 5 seconds, but test fails after 3 seconds
     // -> guarantees that stop is immediate
-    HardStopperThread stopper = new HardStopperThread(monitored, 5000L);
+    AbstractStopperThread stopper = new AbstractStopperThread("theThreadName", () -> monitored.hardStop(), 5000L){};
     stopper.start();
 
     verify(monitored, timeout(3000)).hardStop();
+    assertThat(stopper.getName()).isEqualTo("theThreadName");
   }
 
   @Test
   public void stop_timeout() {
-    doAnswer(new Answer() {
-      @Override
-      public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
-        Thread.sleep(10000L);
-        return null;
-      }
+    doAnswer(invocationOnMock -> {
+      await().atMost(10, TimeUnit.SECONDS).until(() -> false);
+      return null;
     }).when(monitored).hardStop();
 
     // max stop timeout is 100 milliseconds
-    HardStopperThread stopper = new HardStopperThread(monitored, 100L);
+    AbstractStopperThread stopper = new AbstractStopperThread("theThreadName", () -> monitored.hardStop(), 5000L){};
     stopper.start();
 
     verify(monitored, timeout(3000)).hardStop();
+    assertThat(stopper.getName()).isEqualTo("theThreadName");
+  }
+
+  @Test
+  public void stopIt_interrupts_worker() {
+    doAnswer(invocationOnMock -> {
+      await().atMost(10, TimeUnit.SECONDS).until(() -> false);
+      return null;
+    }).when(monitored).hardStop();
+
+    // max stop timeout is 100 milliseconds
+    AbstractStopperThread stopper = new AbstractStopperThread("theThreadName", () -> monitored.hardStop(), 5000L){};
+    stopper.start();
+
+    verify(monitored, timeout(3_000)).hardStop();
+
+    stopper.stopIt();
+    await().atMost(3, TimeUnit.SECONDS).until(() -> !stopper.isAlive());
+    assertThat(stopper.isAlive()).isFalse();
   }
 }
diff --git a/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java b/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java
index bd0689d3ed1..2cbfe3715a7 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java
@@ -19,7 +19,9 @@
  */
 package org.sonar.process;
 
+import java.io.File;
 import java.io.IOException;
+import java.util.Properties;
 import org.apache.commons.io.FileUtils;
 import org.junit.Rule;
 import org.junit.Test;
@@ -31,9 +33,6 @@ import org.sonar.process.Lifecycle.State;
 import org.sonar.process.sharedmemoryfile.ProcessCommands;
 import org.sonar.process.test.StandardProcess;
 
-import java.io.File;
-import java.util.Properties;
-
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
@@ -89,7 +88,7 @@ public class ProcessEntryPointTest {
   }
 
   @Test
-  public void launch_then_request_hard_stop() throws Exception {
+  public void launch_then_request_graceful_stop() throws Exception {
     Props props = createProps();
     final ProcessEntryPoint entryPoint = new ProcessEntryPoint(props, exit, commands);
     final StandardProcess process = new StandardProcess();
@@ -109,9 +108,36 @@ public class ProcessEntryPointTest {
 
     // requests for graceful stop -> waits until down
     // Should terminate before the timeout of 30s
+    entryPoint.stop();
+
+    assertThat(process.getState()).isEqualTo(State.STOPPED);
+    assertThat(process.wasHardStopped()).isEqualTo(false);
+  }
+
+  @Test
+  public void launch_then_request_hard_stop() throws Exception {
+    Props props = createProps();
+    final ProcessEntryPoint entryPoint = new ProcessEntryPoint(props, exit, commands);
+    final StandardProcess process = new StandardProcess();
+
+    Thread runner = new Thread() {
+      @Override
+      public void run() {
+        // starts and waits until terminated
+        entryPoint.launch(process);
+      }
+    };
+    runner.start();
+
+    while (process.getState() != State.STARTED) {
+      Thread.sleep(10L);
+    }
+
+    // requests for stop hardly waiting
     entryPoint.hardStop();
 
     assertThat(process.getState()).isEqualTo(State.STOPPED);
+    assertThat(process.wasHardStopped()).isEqualTo(true);
   }
 
   @Test
@@ -181,6 +207,11 @@ public class ProcessEntryPointTest {
     }
 
     @Override
+    public void stop() {
+
+    }
+
+    @Override
     public void hardStop() {
 
     }
@@ -204,6 +235,11 @@ public class ProcessEntryPointTest {
     }
 
     @Override
+    public void stop() {
+
+    }
+
+    @Override
     public void hardStop() {
 
     }
diff --git a/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java b/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java
index 7db02eff2f1..ef85fd90227 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java
@@ -26,6 +26,7 @@ import org.sonar.process.Lifecycle.State;
 public class StandardProcess implements Monitored {
 
   private State state = State.INIT;
+  private boolean hardStopped = false;
 
   private final Thread daemon = new Thread() {
     @Override
@@ -64,13 +65,21 @@ public class StandardProcess implements Monitored {
     }
   }
 
+  @Override
+  public void stop() {
+    state = State.STOPPING;
+    daemon.interrupt();
+    state = State.STOPPED;
+  }
+
   /**
    * Blocks until stopped
    */
   @Override
   public void hardStop() {
-    state = State.STOPPING;
+    state = State.HARD_STOPPING;
     daemon.interrupt();
+    hardStopped = true;
     state = State.STOPPED;
   }
 
@@ -78,6 +87,10 @@ public class StandardProcess implements Monitored {
     return state;
   }
 
+  public boolean wasHardStopped() {
+    return hardStopped;
+  }
+
   public static void main(String[] args) {
     ProcessEntryPoint entryPoint = ProcessEntryPoint.createForArguments(args);
     entryPoint.launch(new StandardProcess());