SONAR-9739 add integration test for WS api/system/health

3 files changed, 146 insertions, 23 deletions

diff --git a/tests/plugins/server-plugin/src/main/java/ServerStartupLock.java b/tests/plugins/server-plugin/src/main/java/ServerStartupLock.java
index 85aaef2213d..f7c4d7e6863 100644
--- a/tests/plugins/server-plugin/src/main/java/ServerStartupLock.java
+++ b/tests/plugins/server-plugin/src/main/java/ServerStartupLock.java
@@ -20,25 +20,31 @@
 
 import java.io.File;
 import java.util.Optional;
+import org.sonar.api.SonarRuntime;
 import org.sonar.api.Startable;
+import org.sonar.api.ce.ComputeEngineSide;
 import org.sonar.api.config.Configuration;
 import org.sonar.api.server.ServerSide;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 
 @ServerSide
+@ComputeEngineSide
 public class ServerStartupLock implements Startable {
 
-  private final Configuration configuration;
   private static final Logger LOGGER = Loggers.get(ServerStartupLock.class);
 
-  public ServerStartupLock(Configuration configuration) {
+  private final Configuration configuration;
+  private final SonarRuntime runtime;
+
+  public ServerStartupLock(Configuration configuration, SonarRuntime runtime) {
     this.configuration = configuration;
+    this.runtime = runtime;
   }
 
   @Override
   public void start() {
-    Optional<String> path = configuration.get("sonar.test.serverStartupLock.path");
+    Optional<String> path = configuration.get(propertyKey());
     if (path.isPresent()) {
       File lock = new File(path.get());
       try {
@@ -58,4 +64,15 @@ public class ServerStartupLock implements Startable {
   public void stop() {
     // nothing to do
   }
+
+  private String propertyKey() {
+    switch (runtime.getSonarQubeSide()) {
+      case SERVER:
+        return "sonar.web.startupLock.path";
+      case COMPUTE_ENGINE:
+        return "sonar.ce.startupLock.path";
+      default:
+        throw new IllegalArgumentException("Unsupported runtime: " + runtime.getSonarQubeSide());
+    }
+  }
 }
diff --git a/tests/src/test/java/org/sonarqube/tests/Elasticsearch.java b/tests/src/test/java/org/sonarqube/tests/Elasticsearch.java
index a19d9708a92..4bc4620799a 100644
--- a/tests/src/test/java/org/sonarqube/tests/Elasticsearch.java
+++ b/tests/src/test/java/org/sonarqube/tests/Elasticsearch.java
@@ -36,7 +36,7 @@ public class Elasticsearch {
 
   private final int httpPort;
 
-  Elasticsearch(int httpPort) {
+  public Elasticsearch(int httpPort) {
     this.httpPort = httpPort;
   }
 
@@ -55,9 +55,17 @@ public class Elasticsearch {
     putIndexSetting(httpPort, index, "blocks.write", "false");
   }
 
+  public void makeYellow() throws Exception {
+    putIndexSetting(httpPort, "issues", "number_of_replicas", "5");
+  }
+
+  public void makeGreen() throws Exception {
+    putIndexSetting(httpPort, "issues", "number_of_replicas", "0");
+  }
+
   private void putIndexSetting(int searchHttpPort, String index, String key, String value) throws Exception {
     Request.Builder request = new Request.Builder()
-      .url("http://" + InetAddress.getLoopbackAddress().getHostAddress() + ":" + searchHttpPort + "/" + index + "/_settings")
+      .url(baseUrl(searchHttpPort) + index + "/_settings")
       .put(RequestBody.create(MediaType.parse("application/json"), "{" +
         "    \"index\" : {" +
         "        \"" + key + "\" : \"" + value + "\"" +
@@ -67,4 +75,8 @@ public class Elasticsearch {
     Response response = okClient.newCall(request.build()).execute();
     assertThat(response.isSuccessful()).isTrue();
   }
+
+  private String baseUrl(int searchHttpPort) {
+    return "http://" + InetAddress.getLoopbackAddress().getHostAddress() + ":" + searchHttpPort + "/";
+  }
 }
diff --git a/tests/src/test/java/org/sonarqube/tests/serverSystem/SystemStateTest.java b/tests/src/test/java/org/sonarqube/tests/serverSystem/SystemStateTest.java
index b0ac4666913..9559380e2b6 100644
--- a/tests/src/test/java/org/sonarqube/tests/serverSystem/SystemStateTest.java
+++ b/tests/src/test/java/org/sonarqube/tests/serverSystem/SystemStateTest.java
@@ -20,8 +20,10 @@
 package org.sonarqube.tests.serverSystem;
 
 import com.sonar.orchestrator.Orchestrator;
+import com.sonar.orchestrator.util.NetworkUtils;
 import java.io.File;
 import java.io.IOException;
+import java.net.InetAddress;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Supplier;
@@ -32,6 +34,8 @@ import org.junit.rules.DisableOnDebug;
 import org.junit.rules.TemporaryFolder;
 import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
+import org.sonarqube.tests.Elasticsearch;
+import org.sonarqube.ws.WsSystem;
 import org.sonarqube.ws.client.GetRequest;
 import org.sonarqube.ws.client.WsClient;
 import org.sonarqube.ws.client.WsResponse;
@@ -54,32 +58,84 @@ public class SystemStateTest {
   public TestRule safeguard = new DisableOnDebug(Timeout.seconds(300));
 
   @Test
-  public void system_status_becomes_UP_when_web_server_is_started() throws Exception {
+  public void test_status_and_health_during_server_lifecycle() throws Exception {
     try (Commander commander = new Commander()) {
-      commander.startAsync();
-
+      Lock lock = new Lock();
+      commander.start(lock);
       commander.waitFor(() -> commander.webLogsContain("ServerStartupLock - Waiting for file to be deleted"));
-      assertThat(commander.status()).hasValue("STARTING");
 
-      commander.unlock();
+      commander.verifyStatus("STARTING");
+      commander.verifyHealth(WsSystem.Health.RED, "SonarQube webserver is not up");
+
+      lock.unlockWeb();
+      // status is UP as soon as web server is up, whatever the status of Compute Engine
       commander.waitFor(() -> "UP".equals(commander.status().orElse(null)));
+      commander.verifyHealth(WsSystem.Health.RED, "Compute Engine is not operational");
+
+      lock.unlockCe();
+      commander.waitForHealth(WsSystem.Health.GREEN);
+      commander.verifyStatus("UP");
+    }
+  }
+
+  @Test
+  public void test_status_and_health_when_ES_becomes_yellow() throws Exception {
+    try (Commander commander = new Commander()) {
+      commander.start();
+      commander.waitForHealth(WsSystem.Health.GREEN);
+
+      commander.makeElasticsearchYellow();
+      commander.waitForHealth(WsSystem.Health.YELLOW, "Elasticsearch status is YELLOW");
+      commander.verifyStatus("UP");
+
+      commander.makeElasticsearchGreen();
+      commander.waitForHealth(WsSystem.Health.GREEN);
+      // status does not change after being UP
+      commander.verifyStatus("UP");
+    }
+  }
+
+  private class Lock {
+    private final File webFile;
+    private final File ceFile;
+
+    Lock() throws Exception {
+      webFile = temp.newFile();
+      ceFile = temp.newFile();
+    }
+
+    void unlockWeb() throws IOException {
+      FileUtils.forceDelete(webFile);
+    }
+
+    void unlockCe() throws IOException {
+      FileUtils.forceDelete(ceFile);
     }
   }
 
   private class Commander implements AutoCloseable {
-    private final File lock = temp.newFile();
-    private final Orchestrator orchestrator = Orchestrator.builderEnv()
-      .addPlugin(pluginArtifact("server-plugin"))
-      .setServerProperty("sonar.test.serverStartupLock.path", lock.getCanonicalPath())
-      .build();
+    private final int esHttpPort = NetworkUtils.getNextAvailablePort(InetAddress.getLoopbackAddress());
+    private Orchestrator orchestrator;
     private Thread starter;
+    private Elasticsearch elasticsearch;
 
-    Commander() throws Exception {
-
+    void start() throws Exception {
+      Lock lock = new Lock();
+      start(lock);
+      lock.unlockWeb();
+      lock.unlockCe();
     }
 
-    void startAsync() {
-      checkState(starter == null);
+    void start(Lock lock) {
+      checkState(orchestrator == null);
+      orchestrator = Orchestrator.builderEnv()
+        .addPlugin(pluginArtifact("server-plugin"))
+        .setServerProperty("sonar.web.startupLock.path", lock.webFile.getAbsolutePath())
+        .setServerProperty("sonar.ce.startupLock.path", lock.ceFile.getAbsolutePath())
+        .setServerProperty("sonar.search.httpPort", "" + esHttpPort)
+        .build();
+      elasticsearch = new Elasticsearch(esHttpPort);
+
       starter = new Thread(orchestrator::start);
       starter.start();
       while (orchestrator.getServer() == null) {
@@ -87,10 +143,6 @@ public class SystemStateTest {
       }
     }
 
-    void unlock() throws IOException {
-      FileUtils.forceDelete(lock);
-    }
-
     boolean webLogsContain(String message) {
       try {
         return FileUtils.readFileToString(orchestrator.getServer().getWebLogs()).contains(message);
@@ -129,6 +181,48 @@ public class SystemStateTest {
       return Optional.empty();
     }
 
+    void verifyStatus(String expectedStatus) {
+      assertThat(status()).hasValue(expectedStatus);
+    }
+
+    Optional<WsSystem.Health> health() {
+      Optional<WsSystem.HealthResponse> response = healthResponse();
+      return response.map(WsSystem.HealthResponse::getHealth);
+    }
+
+    Optional<WsSystem.HealthResponse> healthResponse() {
+      if (orchestrator.getServer() != null) {
+        WsClient wsClient = newWsClient(orchestrator);
+        try {
+          return Optional.of(wsClient.system().health());
+        } catch (Exception e) {
+          // server does not accept connections
+        }
+      }
+      return Optional.empty();
+    }
+
+    void waitForHealth(WsSystem.Health expectedHealth, String... expectedMessages) {
+      waitFor(() -> expectedHealth == health().orElse(null));
+      verifyHealth(expectedHealth, expectedMessages);
+    }
+
+    void verifyHealth(WsSystem.Health expectedHealth, String... expectedMessages) {
+      WsSystem.HealthResponse response = healthResponse().get();
+      assertThat(response.getHealth()).isEqualTo(expectedHealth);
+      assertThat(response.getCausesList())
+        .extracting(WsSystem.Cause::getMessage)
+        .containsExactlyInAnyOrder(expectedMessages);
+    }
+
+    void makeElasticsearchYellow() throws Exception {
+      elasticsearch.makeYellow();
+    }
+
+    void makeElasticsearchGreen() throws Exception {
+      elasticsearch.makeGreen();
+    }
+
     @Override
     public void close() {
       if (starter != null) {