SONAR-8435 log "SonarQube is up" if all processes are operational

which implies:
1/ to distinguish from Monitored#getStatus() = UP and OPERATIONAL
2/ to have an option of Monitor to wait on process's status to be OPERATIONAL
3/ every Monitored implementation must return OPERATIONAL rather than UP if then don't make a distinction between the two

8 files changed, 69 insertions, 30 deletions

diff --git a/server/sonar-process/src/main/java/org/sonar/process/Lifecycle.java b/server/sonar-process/src/main/java/org/sonar/process/Lifecycle.java
index 984aaafdb24..25ca71f59fc 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/Lifecycle.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/Lifecycle.java
@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
 
 import static org.sonar.process.Lifecycle.State.HARD_STOPPING;
 import static org.sonar.process.Lifecycle.State.INIT;
+import static org.sonar.process.Lifecycle.State.OPERATIONAL;
 import static org.sonar.process.Lifecycle.State.RESTARTING;
 import static org.sonar.process.Lifecycle.State.STARTED;
 import static org.sonar.process.Lifecycle.State.STARTING;
@@ -44,7 +45,7 @@ public class Lifecycle {
   private static final Logger LOG = LoggerFactory.getLogger(Lifecycle.class);
 
   public enum State {
-    INIT, STARTING, STARTED, RESTARTING, STOPPING, HARD_STOPPING, STOPPED
+    INIT, STARTING, STARTED, OPERATIONAL, RESTARTING, STOPPING, HARD_STOPPING, STOPPED
   }
 
   private static final Map<State, Set<State>> TRANSITIONS = buildTransitions();
@@ -60,7 +61,8 @@ public class Lifecycle {
     Map<State, Set<State>> res = new EnumMap<>(State.class);
     res.put(INIT, toSet(STARTING));
     res.put(STARTING, toSet(STARTED, STOPPING, HARD_STOPPING));
-    res.put(STARTED, toSet(RESTARTING, STOPPING, HARD_STOPPING));
+    res.put(STARTED, toSet(OPERATIONAL, RESTARTING, STOPPING, HARD_STOPPING));
+    res.put(OPERATIONAL, toSet(RESTARTING, STOPPING, HARD_STOPPING));
     res.put(RESTARTING, toSet(STARTING, HARD_STOPPING));
     res.put(STOPPING, toSet(STOPPED));
     res.put(HARD_STOPPING, toSet(STOPPED));
diff --git a/server/sonar-process/src/main/java/org/sonar/process/Monitored.java b/server/sonar-process/src/main/java/org/sonar/process/Monitored.java
index 837c658cf3b..b37d929e54d 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/Monitored.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/Monitored.java
@@ -27,13 +27,13 @@ public interface Monitored {
   void start();
 
   /**
-   * {@link Status#UP} if the process is done starting, {@link Status#FAILED} if the process failed to start,
-   * {@link Status#DOWN} otherwise.
+   * @return {@link Status#UP} if the process is done starting, {@link Status#OPERATIONAL} if the service is operation,
+   *         {@link Status#FAILED} if the process failed to start, {@link Status#DOWN} otherwise.
    */
   Status getStatus();
 
   enum Status {
-    UP, DOWN, FAILED
+    DOWN, UP, OPERATIONAL, FAILED
   }
 
   /**
diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java
index dffa067fd2c..15190a41dcc 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessEntryPoint.java
@@ -95,22 +95,7 @@ public class ProcessEntryPoint implements Stoppable {
 
     Logger logger = LoggerFactory.getLogger(getClass());
     try {
-      logger.info("Starting " + getKey());
-      Runtime.getRuntime().addShutdownHook(shutdownHook);
-      stopWatcher.start();
-
-      monitored.start();
-      Monitored.Status status = waitForNotDownStatus();
-      if (status == Monitored.Status.UP) {
-        // notify monitor that process is ready
-        commands.setUp();
-
-        if (lifecycle.tryToMoveTo(Lifecycle.State.STARTED)) {
-          monitored.awaitStop();
-        }
-      } else {
-        stop();
-      }
+      launch(logger);
     } catch (Exception e) {
       logger.warn("Fail to start " + getKey(), e);
     } finally {
@@ -118,6 +103,30 @@ public class ProcessEntryPoint implements Stoppable {
     }
   }
 
+  private void launch(Logger logger) throws InterruptedException {
+    logger.info("Starting " + getKey());
+    Runtime.getRuntime().addShutdownHook(shutdownHook);
+    stopWatcher.start();
+
+    monitored.start();
+    Monitored.Status status = waitForNotDownStatus();
+    if (status == Monitored.Status.UP || status == Monitored.Status.OPERATIONAL) {
+      // notify monitor that process is ready
+      commands.setUp();
+
+      if (lifecycle.tryToMoveTo(Lifecycle.State.STARTED)) {
+        Monitored.Status newStatus = waitForOperational(status);
+        if (newStatus == Monitored.Status.OPERATIONAL && lifecycle.tryToMoveTo(Lifecycle.State.OPERATIONAL)) {
+          commands.setOperational();
+        }
+
+        monitored.awaitStop();
+      }
+    } else {
+      stop();
+    }
+  }
+
   private Monitored.Status waitForNotDownStatus() throws InterruptedException {
     Monitored.Status status = Monitored.Status.DOWN;
     while (status == Monitored.Status.DOWN) {
@@ -127,6 +136,16 @@ public class ProcessEntryPoint implements Stoppable {
     return status;
   }
 
+  private Monitored.Status waitForOperational(Monitored.Status currentStatus) throws InterruptedException {
+    Monitored.Status status = currentStatus;
+    // wait for operation or stop waiting if going to OPERATIONAL failed
+    while (status != Monitored.Status.OPERATIONAL && status != Monitored.Status.FAILED) {
+      status = monitored.getStatus();
+      Thread.sleep(20L);
+    }
+    return status;
+  }
+
   boolean isStarted() {
     return lifecycle.getState() == Lifecycle.State.STARTED;
   }
diff --git a/server/sonar-process/src/test/java/org/sonar/process/LifecycleTest.java b/server/sonar-process/src/test/java/org/sonar/process/LifecycleTest.java
index dbb233e474e..fc773dde56a 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/LifecycleTest.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/LifecycleTest.java
@@ -28,6 +28,7 @@ import org.junit.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.sonar.process.Lifecycle.State;
 import static org.sonar.process.Lifecycle.State.INIT;
+import static org.sonar.process.Lifecycle.State.OPERATIONAL;
 import static org.sonar.process.Lifecycle.State.RESTARTING;
 import static org.sonar.process.Lifecycle.State.STARTED;
 import static org.sonar.process.Lifecycle.State.STARTING;
@@ -75,11 +76,11 @@ public class LifecycleTest {
   }
 
   @Test
-  public void can_move_to_STOPPING_from_STARTING_and_STARTED_only() {
+  public void can_move_to_STOPPING_from_STARTING_STARTED_OPERATIONAL_only() {
     for (State state : values()) {
       TestLifeCycleListener listener = new TestLifeCycleListener();
       boolean tryToMoveTo = newLifeCycle(state, listener).tryToMoveTo(STOPPING);
-      if (state == STARTING || state == STARTED) {
+      if (state == STARTING || state == STARTED  || state == OPERATIONAL) {
         assertThat(tryToMoveTo).describedAs("from state " + state).isTrue();
         assertThat(listener.getTransitions()).containsOnly(new Transition(state, STOPPING));
       } else {
@@ -90,6 +91,21 @@ public class LifecycleTest {
   }
 
   @Test
+  public void can_move_to_OPERATIONAL_from_STARTED_only() {
+    for (State state : values()) {
+      TestLifeCycleListener listener = new TestLifeCycleListener();
+      boolean tryToMoveTo = newLifeCycle(state, listener).tryToMoveTo(OPERATIONAL);
+      if (state == STARTED) {
+        assertThat(tryToMoveTo).describedAs("from state " + state).isTrue();
+        assertThat(listener.getTransitions()).containsOnly(new Transition(state, OPERATIONAL));
+      } else {
+        assertThat(tryToMoveTo).describedAs("from state " + state).isFalse();
+        assertThat(listener.getTransitions()).isEmpty();
+      }
+    }
+  }
+
+  @Test
   public void can_move_to_STARTING_from_RESTARTING() {
     TestLifeCycleListener listener = new TestLifeCycleListener();
     assertThat(newLifeCycle(RESTARTING, listener).tryToMoveTo(STARTING)).isTrue();
@@ -104,10 +120,12 @@ public class LifecycleTest {
         return newLifeCycle(INIT, state, listeners);
       case STARTED:
         return newLifeCycle(STARTING, state, listeners);
-      case RESTARTING:
+      case OPERATIONAL:
         return newLifeCycle(STARTED, state, listeners);
+      case RESTARTING:
+        return newLifeCycle(OPERATIONAL, state, listeners);
       case STOPPING:
-        return newLifeCycle(STARTED, state, listeners);
+        return newLifeCycle(OPERATIONAL, state, listeners);
       case HARD_STOPPING:
         return newLifeCycle(STARTING, state, listeners);
       case STOPPED:
diff --git a/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java b/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java
index 02c07db7c66..3b045b6a0dc 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/ProcessEntryPointTest.java
@@ -174,7 +174,7 @@ public class ProcessEntryPointTest {
 
     @Override
     public Status getStatus() {
-      return Status.UP;
+      return Status.OPERATIONAL;
     }
 
     @Override
diff --git a/server/sonar-process/src/test/java/org/sonar/process/test/HttpProcess.java b/server/sonar-process/src/test/java/org/sonar/process/test/HttpProcess.java
index d11b173400e..93110668414 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/test/HttpProcess.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/test/HttpProcess.java
@@ -90,13 +90,13 @@ public class HttpProcess implements Monitored {
   @Override
   public Status getStatus() {
     if (ready) {
-      return Status.UP;
+      return Status.OPERATIONAL;
     }
     if (server.isStarted()) {
       ready = true;
       writeTimeToFile("readyAt");
     }
-    return ready ? Status.UP : Status.DOWN;
+    return ready ? Status.OPERATIONAL : Status.DOWN;
   }
 
   @Override
diff --git a/server/sonar-process/src/test/java/org/sonar/process/test/InfiniteTerminationProcess.java b/server/sonar-process/src/test/java/org/sonar/process/test/InfiniteTerminationProcess.java
index 47edc81a47c..b69c998eca6 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/test/InfiniteTerminationProcess.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/test/InfiniteTerminationProcess.java
@@ -52,7 +52,7 @@ public class InfiniteTerminationProcess implements Monitored {
 
   @Override
   public Status getStatus() {
-    return state == State.STARTED ? Status.UP : Status.DOWN;
+    return state == State.STARTED ? Status.OPERATIONAL : Status.DOWN;
   }
 
   @Override
diff --git a/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java b/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java
index 2c7cf0aaeff..e5086d87bf0 100644
--- a/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/test/StandardProcess.java
@@ -52,7 +52,7 @@ public class StandardProcess implements Monitored {
 
   @Override
   public Status getStatus() {
-    return state == State.STARTED ? Status.UP : Status.DOWN;
+    return state == State.STARTED ? Status.OPERATIONAL : Status.DOWN;
   }
 
   @Override