SONAR-9741 add integration tests

6 files changed, 211 insertions, 49 deletions

diff --git a/tests/src/test/java/org/sonarqube/tests/LogsTailer.java b/tests/src/test/java/org/sonarqube/tests/LogsTailer.java
index 06f4d7d189a..e367ba3d7cc 100644
--- a/tests/src/test/java/org/sonarqube/tests/LogsTailer.java
+++ b/tests/src/test/java/org/sonarqube/tests/LogsTailer.java
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CountDownLatch;
 import java.util.function.Consumer;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import org.apache.commons.io.input.Tailer;
 import org.apache.commons.io.input.TailerListenerAdapter;
@@ -53,7 +54,7 @@ public class LogsTailer implements AutoCloseable {
   }
 
   @Override
-  public void close() throws Exception {
+  public void close() {
     for (Tailer tailer : tailers) {
       tailer.stop();
     }
@@ -78,19 +79,17 @@ public class LogsTailer implements AutoCloseable {
       return this;
     }
 
+    /**
+     * Adds a consumer that is called on each new line appended
+     * to the files.
+     * Note that the consumer {@link Content} allows to keep
+     * all past logs in memory.
+     */
     public Builder addConsumer(Consumer<String> consumer) {
       this.consumers.add(consumer);
       return this;
     }
 
-    public Builder doOnFind(String text, Runnable runnable) {
-      return addConsumer(log -> {
-        if (log.contains(text)) {
-          runnable.run();
-        }
-      });
-    }
-
     public LogsTailer build() {
       return new LogsTailer(this);
     }
@@ -159,4 +158,35 @@ public class LogsTailer implements AutoCloseable {
       logConsumer.remove(consumer);
     }
   }
+
+  public static class Content implements Consumer<String> {
+    private final List<String> lines = Collections.synchronizedList(new ArrayList<>());
+
+    @Override
+    public void accept(String s) {
+      lines.add(s);
+    }
+
+    public boolean hasText(String text) {
+      synchronized (lines) {
+        for (String line : lines) {
+          if (line.contains(text)) {
+            return true;
+          }
+        }
+      }
+      return false;
+    }
+
+    public boolean hasLineMatching(Pattern pattern) {
+      synchronized (lines) {
+        for (String line : lines) {
+          if (pattern.matcher(line).matches()) {
+            return true;
+          }
+        }
+      }
+      return false;
+    }
+  }
 }
diff --git a/tests/src/test/java/org/sonarqube/tests/ce/CeShutdownTest.java b/tests/src/test/java/org/sonarqube/tests/ce/CeShutdownTest.java
index 29c237ca7ca..30ab04a55ca 100644
--- a/tests/src/test/java/org/sonarqube/tests/ce/CeShutdownTest.java
+++ b/tests/src/test/java/org/sonarqube/tests/ce/CeShutdownTest.java
@@ -23,6 +23,7 @@ import com.sonar.orchestrator.Orchestrator;
 import com.sonar.orchestrator.build.SonarScanner;
 import java.io.File;
 import java.io.IOException;
+import java.util.regex.Pattern;
 import org.apache.commons.io.FileUtils;
 import org.junit.Ignore;
 import org.junit.Rule;
@@ -69,7 +70,7 @@ public class CeShutdownTest {
       try (LogsTailer.Watch watch = ce.logs().watch("Process [ce] is stopped")) {
         ce.resumeTask();
         watch.waitForLog();
-        assertThat(ce.isTaskFinished()).isTrue();
+        assertThat(ce.hasTaskFinishedSuccessfully()).isTrue();
         assertThat(ce.hasErrorLogs()).isFalse();
       }
     }
@@ -98,7 +99,7 @@ public class CeShutdownTest {
       // finish successfully
       try (LogsTailer.Watch watch = ce.logs().watch("Process [ce] is stopped")) {
         watch.waitForLog();
-        assertThat(ce.isTaskFinished()).isTrue();
+        assertThat(ce.hasTaskFinishedSuccessfully()).isTrue();
         assertThat(ce.hasErrorLogs()).isTrue();
       }
     }
@@ -110,6 +111,7 @@ public class CeShutdownTest {
     private final WsClient adminWsClient;
     private Thread stopper;
     private final LogsTailer logsTailer;
+    private final LogsTailer.Content content = new LogsTailer.Content();
 
     ComputeEngine() throws Exception {
       pauseFile = temp.newFile();
@@ -125,6 +127,7 @@ public class CeShutdownTest {
       logsTailer = LogsTailer.builder()
         .addFile(orchestrator.getServer().getCeLogs())
         .addFile(orchestrator.getServer().getAppLogs())
+        .addConsumer(content)
         .build();
     }
 
@@ -144,14 +147,12 @@ public class CeShutdownTest {
       return adminWsClient.ce().activityStatus(ActivityStatusWsRequest.newBuilder().build()).getInProgress();
     }
 
-    boolean isTaskFinished() throws Exception {
-      String ceLogs = FileUtils.readFileToString(orchestrator.getServer().getCeLogs());
-      return ceLogs.contains("Executed task | project=foo | type=REPORT");
+    boolean hasTaskFinishedSuccessfully() throws Exception {
+      return content.hasLineMatching(Pattern.compile(".* INFO .*Executed task \\| project=foo \\| type=REPORT.*"));
     }
 
     boolean hasErrorLogs() throws IOException {
-      String ceLogs = FileUtils.readFileToString(orchestrator.getServer().getCeLogs());
-      return ceLogs.contains(" ERROR ");
+      return content.hasText(" ERROR ");
     }
 
     /**
@@ -171,6 +172,7 @@ public class CeShutdownTest {
       if (orchestrator != null) {
         orchestrator.stop();
       }
+      logsTailer.close();
     }
   }
 }
diff --git a/tests/src/test/java/org/sonarqube/tests/cluster/Cluster.java b/tests/src/test/java/org/sonarqube/tests/cluster/Cluster.java
index 9340ce53d8a..74efbf7b53a 100644
--- a/tests/src/test/java/org/sonarqube/tests/cluster/Cluster.java
+++ b/tests/src/test/java/org/sonarqube/tests/cluster/Cluster.java
@@ -27,6 +27,7 @@ import java.util.function.Consumer;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
 import org.slf4j.LoggerFactory;
+import util.ItUtils;
 
 import static java.util.stream.Collectors.joining;
 
@@ -83,14 +84,14 @@ class Cluster implements AutoCloseable {
     return nodes.stream().filter(n -> n.getConfig().getType() == NodeConfig.NodeType.APPLICATION);
   }
 
-  Stream<Node> getSearchNodes() {
-    return nodes.stream().filter(n -> n.getConfig().getType() == NodeConfig.NodeType.SEARCH);
-  }
-
   Node getAppNode(int index) {
     return getAppNodes().skip(index).findFirst().orElseThrow(IllegalArgumentException::new);
   }
 
+  Stream<Node> getSearchNodes() {
+    return nodes.stream().filter(n -> n.getConfig().getType() == NodeConfig.NodeType.SEARCH);
+  }
+
   Node getSearchNode(int index) {
     return getSearchNodes().skip(index).findFirst().orElseThrow(IllegalArgumentException::new);
   }
@@ -121,6 +122,7 @@ class Cluster implements AutoCloseable {
     if (node.getName().isPresent()) {
       builder.setServerProperty("sonar.cluster.node.name", node.getName().get());
     }
+    builder.addPlugin(ItUtils.pluginArtifact("server-plugin"));
     builder.setStartupLogWatcher(logLine -> true);
     return builder;
   }
diff --git a/tests/src/test/java/org/sonarqube/tests/cluster/ClusterTest.java b/tests/src/test/java/org/sonarqube/tests/cluster/ClusterTest.java
index 0cf07e3fda5..159375c9c4e 100644
--- a/tests/src/test/java/org/sonarqube/tests/cluster/ClusterTest.java
+++ b/tests/src/test/java/org/sonarqube/tests/cluster/ClusterTest.java
@@ -22,18 +22,23 @@ package org.sonarqube.tests.cluster;
 import com.sonar.orchestrator.Orchestrator;
 import com.sonar.orchestrator.OrchestratorBuilder;
 import com.sonar.orchestrator.db.DefaultDatabase;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.BinaryOperator;
 import java.util.function.Consumer;
 import java.util.stream.IntStream;
+import org.apache.commons.io.FileUtils;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.DisableOnDebug;
 import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
 import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
+import org.sonarqube.ws.WsSystem;
 
 import static com.google.common.base.Preconditions.checkState;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -49,6 +54,9 @@ public class ClusterTest {
   @Rule
   public ExpectedException expectedException = ExpectedException.none();
 
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
   @BeforeClass
   public static void initDbSchema() throws Exception {
     Orchestrator orchestrator = Orchestrator.builderEnv()
@@ -65,10 +73,9 @@ public class ClusterTest {
   public void test_high_availability_topology() throws Exception {
     try (Cluster cluster = newCluster(3, 2)) {
       cluster.getNodes().forEach(Node::start);
-      cluster.getAppNodes().forEach(Node::waitForStatusUp);
 
-      // TODO verify cluster health to be green
-      // TODO verify that ES cluster is green
+      cluster.getAppNode(0).waitForHealthGreen();
+      cluster.getAppNodes().forEach(node -> assertThat(node.getStatus()).hasValue(WsSystem.Status.UP));
 
       cluster.getNodes().forEach(node -> {
         node.assertThatProcessesAreUp();
@@ -76,6 +83,8 @@ public class ClusterTest {
         assertThat(node.anyLogsContain("MessageException")).isFalse();
       });
 
+      verifyGreenHealthOfNodes(cluster);
+
       // verify that there's a single web startup leader
       Node startupLeader = cluster.getAppNodes()
         .filter(Node::isStartupLeader)
@@ -100,6 +109,21 @@ public class ClusterTest {
     }
   }
 
+  private void verifyGreenHealthOfNodes(Cluster cluster) {
+    WsSystem.HealthResponse health = cluster.getAppNode(0).getHealth().get();
+    cluster.getNodes().forEach(node -> {
+      WsSystem.Node healthNode = health.getNodes().getNodesList().stream()
+        .filter(n -> n.getPort() == node.getConfig().getHzPort())
+        .findFirst()
+        .orElseThrow(() -> new IllegalStateException("Node with port " + node.getConfig().getHzPort() + " not found in api/system/health"));
+      // TODO assertions to be improved
+      assertThat(healthNode.getStartedAt()).isNotEmpty();
+      assertThat(healthNode.getHost()).isNotEmpty();
+      assertThat(healthNode.getCausesCount()).isEqualTo(0);
+      assertThat(healthNode.getHealth()).isEqualTo(WsSystem.Health.GREEN);
+    });
+  }
+
   @Test
   public void minimal_cluster_is_2_search_and_1_application_nodes() throws Exception {
     try (Cluster cluster = newCluster(2, 1)) {
@@ -108,12 +132,16 @@ public class ClusterTest {
       Node app = cluster.getAppNode(0);
       app.waitForStatusUp();
       app.waitForCeLogsContain("Compute Engine is operational");
+
+      app.waitForHealth(WsSystem.Health.YELLOW);
+      WsSystem.HealthResponse health = app.getHealth().orElseThrow(() -> new IllegalStateException("Health is not available"));
+      assertThat(health.getCausesList()).extracting(WsSystem.Cause::getMessage)
+        .contains("There should be at least three search nodes")
+        .contains("There should be at least two application nodes");
+
       assertThat(app.isStartupLeader()).isTrue();
       assertThat(app.hasStartupLeaderOperations()).isTrue();
 
-      // TODO verify cluster health to be yellow
-      // TODO verify that ES cluster is yellow
-
       cluster.getNodes().forEach(node -> {
         assertThat(node.anyLogsContain(" ERROR ")).isFalse();
         node.assertThatProcessesAreUp();
@@ -145,8 +173,6 @@ public class ClusterTest {
       app.waitForStatusUp();
       assertThat(app.isStartupLeader()).isTrue();
       assertThat(app.hasStartupLeaderOperations()).isTrue();
-      // TODO verify cluster health to be yellow
-      // TODO verify that ES cluster is yellow
 
       // no errors
       cluster.getNodes().forEach(node -> {
@@ -222,6 +248,49 @@ public class ClusterTest {
     }
   }
 
+  @Test
+  @Ignore("WS api/system/health returns 500")
+  public void health_becomes_RED_when_all_search_nodes_go_down() throws Exception {
+    try (Cluster cluster = newCluster(2, 1)) {
+      cluster.getNodes().forEach(Node::start);
+
+      Node app = cluster.getAppNode(0);
+      app.waitForHealth(WsSystem.Health.YELLOW);
+
+      cluster.getSearchNodes().forEach(Node::stop);
+
+      app.waitForHealth(WsSystem.Health.RED);
+      assertThat(app.getHealth().get().getCausesList()).extracting(WsSystem.Cause::getMessage)
+        .contains("Elasticsearch status is RED");
+    }
+  }
+
+  @Test
+  public void health_ws_is_available_when_server_is_starting() throws Exception {
+    File startupLock = temp.newFile();
+    FileUtils.touch(startupLock);
+
+    try (Cluster cluster = newCluster(2, 0)) {
+      // add an application node that pauses during startup
+      NodeConfig appConfig = NodeConfig.newApplicationConfig()
+        .addConnectionToBus(cluster.getSearchNode(0).getConfig())
+        .addConnectionToSearch(cluster.getSearchNode(0).getConfig());
+      Node appNode = cluster.addNode(appConfig, b -> b.setServerProperty("sonar.web.startupLock.path", startupLock.getAbsolutePath()));
+
+      cluster.getNodes().forEach(Node::start);
+
+      appNode.waitFor(node -> WsSystem.Status.STARTING == node.getStatus().orElse(null));
+
+      // WS answers whereas server is still not started
+      assertThat(appNode.getHealth().get().getHealth()).isEqualTo(WsSystem.Health.RED);
+
+      // just to be sure, verify that server is still being started
+      assertThat(appNode.getStatus()).hasValue(WsSystem.Status.STARTING);
+
+      startupLock.delete();
+    }
+  }
+
   /**
    * Used to have non-blocking {@link Node#start()}. Orchestrator considers
    * node to be up as soon as the first log is generated.
diff --git a/tests/src/test/java/org/sonarqube/tests/cluster/Node.java b/tests/src/test/java/org/sonarqube/tests/cluster/Node.java
index 03dd4daf456..7826325d34d 100644
--- a/tests/src/test/java/org/sonarqube/tests/cluster/Node.java
+++ b/tests/src/test/java/org/sonarqube/tests/cluster/Node.java
@@ -24,18 +24,26 @@ import com.sonar.orchestrator.Orchestrator;
 import java.io.File;
 import java.io.IOException;
 import java.net.ServerSocket;
-import java.util.Map;
+import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 import javax.annotation.Nullable;
 import org.apache.commons.io.FileUtils;
+import org.sonarqube.tests.LogsTailer;
+import org.sonarqube.ws.WsSystem;
+import org.sonarqube.ws.client.WsClient;
 import util.ItUtils;
 
+import static com.google.common.base.Preconditions.checkState;
 import static org.assertj.core.api.Assertions.assertThat;
 
 class Node {
 
   private final NodeConfig config;
   private final Orchestrator orchestrator;
+  private LogsTailer logsTailer;
+  private final LogsTailer.Content content = new LogsTailer.Content();
 
   Node(NodeConfig config, Orchestrator orchestrator) {
     this.config = config;
@@ -53,10 +61,20 @@ class Node {
    */
   void start() {
     orchestrator.start();
+    logsTailer = LogsTailer.builder()
+      .addFile(orchestrator.getServer().getWebLogs())
+      .addFile(orchestrator.getServer().getCeLogs())
+      .addFile(orchestrator.getServer().getEsLogs())
+      .addFile(orchestrator.getServer().getAppLogs())
+      .addConsumer(content)
+      .build();
   }
 
   void stop() {
     orchestrator.stop();
+    if (logsTailer != null) {
+      logsTailer.close();
+    }
   }
 
   void cleanUpLogs() {
@@ -77,22 +95,62 @@ class Node {
   }
 
   void waitForStatusUp() {
-    waitForStatus("UP");
+    waitFor(node -> WsSystem.Status.UP == node.getStatus().orElse(null));
+  }
+
+  /**
+   * Waiting for health to be green... or yellow on the boxes that
+   * have less than 15% of free disk space. In that case Elasticsearch
+   * can't build shard replicas so it is yellow.
+   */
+  void waitForHealthGreen() {
+    waitFor(node -> {
+      Optional<WsSystem.HealthResponse> health = node.getHealth();
+      if (!health.isPresent()) {
+        return false;
+      }
+      if (health.get().getHealth() == WsSystem.Health.GREEN) {
+        return true;
+      }
+      if (health.get().getHealth() == WsSystem.Health.YELLOW) {
+        List<WsSystem.Cause> causes = health.get().getCausesList();
+        return causes.size() == 1 && "Elasticsearch status is YELLOW".equals(causes.get(0).getMessage());
+      }
+      return false;
+    });
+  }
+
+  void waitForHealth(WsSystem.Health expectedHealth) {
+    waitFor(node -> expectedHealth.equals(node.getHealth().map(WsSystem.HealthResponse::getHealth).orElse(null)));
+  }
+
+  Optional<WsSystem.Status> getStatus() {
+    checkState(config.getType() == NodeConfig.NodeType.APPLICATION);
+    if (orchestrator.getServer() == null) {
+      return Optional.empty();
+    }
+    try {
+      return Optional.ofNullable(ItUtils.newAdminWsClient(orchestrator).system().status().getStatus());
+    } catch (Exception e) {
+      return Optional.empty();
+    }
   }
 
-  void waitForStatus(String expectedStatus) {
-    String status = null;
+  Optional<WsSystem.HealthResponse> getHealth() {
+    checkState(config.getType() == NodeConfig.NodeType.APPLICATION);
+    if (orchestrator.getServer() == null) {
+      return Optional.empty();
+    }
     try {
-      while (!expectedStatus.equals(status)) {
-        if (orchestrator.getServer() != null) {
-          try {
-            Map<String, Object> json = ItUtils.jsonToMap(orchestrator.getServer().newHttpCall("api/system/status").executeUnsafely().getBodyAsString());
-            status = (String) json.get("status");
-          } catch (Exception e) {
-            // ignored
-          }
-        }
+      return Optional.ofNullable(ItUtils.newAdminWsClient(orchestrator).system().health());
+    } catch (Exception e) {
+      return Optional.empty();
+    }
+  }
 
+  void waitFor(Predicate<Node> predicate) {
+    try {
+      while (!predicate.test(this)) {
         Thread.sleep(500);
       }
     } catch (InterruptedException e) {
@@ -144,13 +202,7 @@ class Node {
   }
 
   boolean anyLogsContain(String message) {
-    if (orchestrator.getServer() == null) {
-      return false;
-    }
-    return fileContains(orchestrator.getServer().getAppLogs(), message) ||
-      fileContains(orchestrator.getServer().getWebLogs(), message) ||
-      fileContains(orchestrator.getServer().getEsLogs(), message) ||
-      fileContains(orchestrator.getServer().getCeLogs(), message);
+    return content.hasText(message);
   }
 
   private boolean webLogsContain(String message) {
@@ -182,4 +234,8 @@ class Node {
     }
   }
 
+  public WsClient wsClient() {
+    checkState(config.getType() == NodeConfig.NodeType.APPLICATION);
+    return ItUtils.newAdminWsClient(orchestrator);
+  }
 }
diff --git a/tests/src/test/java/org/sonarqube/tests/serverSystem/SystemStateTest.java b/tests/src/test/java/org/sonarqube/tests/serverSystem/SystemStateTest.java
index e942a432e96..b61b5ba8237 100644
--- a/tests/src/test/java/org/sonarqube/tests/serverSystem/SystemStateTest.java
+++ b/tests/src/test/java/org/sonarqube/tests/serverSystem/SystemStateTest.java
@@ -38,6 +38,7 @@ import org.sonarqube.ws.WsSystem;
 import org.sonarqube.ws.client.WsClient;
 
 import static com.google.common.base.Preconditions.checkState;
+import static java.util.stream.Collectors.joining;
 import static org.assertj.core.api.Assertions.assertThat;
 import static util.ItUtils.newWsClient;
 import static util.ItUtils.pluginArtifact;
@@ -201,7 +202,9 @@ public class SystemStateTest {
 
     void verifyHealth(WsSystem.Health expectedHealth, String... expectedMessages) {
       WsSystem.HealthResponse response = healthResponse().get();
-        assertThat(response.getHealth()).isEqualTo(expectedHealth);
+      assertThat(response.getHealth())
+        .as(response.getCausesList().stream().map(WsSystem.Cause::getMessage).collect(joining(",")))
+        .isEqualTo(expectedHealth);
       assertThat(response.getCausesList())
         .extracting(WsSystem.Cause::getMessage)
         .containsExactlyInAnyOrder(expectedMessages);