SONAR-9741 harden ES status check when no node can be contacted

or any other Exception occuring while checking ES cluster status

3 files changed, 65 insertions, 17 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/health/EsStatusCheck.java b/server/sonar-server/src/main/java/org/sonar/server/health/EsStatusCheck.java
index 1110a9e362e..7c4c0bb57a1 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/health/EsStatusCheck.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/health/EsStatusCheck.java
@@ -20,11 +20,15 @@
 package org.sonar.server.health;
 
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 import org.sonar.server.es.EsClient;
 
 import static org.sonar.server.health.Health.newHealthCheckBuilder;
 
-public abstract class EsStatusCheck {
+abstract class EsStatusCheck {
+  private static final Logger LOG = Loggers.get(EsStatusCheck.class);
+
   private static final Health YELLOW_HEALTH = newHealthCheckBuilder()
     .setStatus(Health.Status.YELLOW)
     .addCause("Elasticsearch status is YELLOW")
@@ -33,23 +37,33 @@ public abstract class EsStatusCheck {
     .setStatus(Health.Status.RED)
     .addCause("Elasticsearch status is RED")
     .build();
-  protected final EsClient esClient;
+  private static final Health RED_HEALTH_EXCEPTION_OCCURED = newHealthCheckBuilder()
+    .setStatus(Health.Status.RED)
+    .addCause("Elasticsearch status is RED (unavailable)")
+    .build();
+
+  private final EsClient esClient;
 
   EsStatusCheck(EsClient esClient) {
     this.esClient = esClient;
   }
 
   Health checkEsStatus() {
-    ClusterHealthStatus esStatus = esClient.prepareClusterStats().get().getStatus();
-    switch (esStatus) {
-      case GREEN:
-        return Health.GREEN;
-      case YELLOW:
-        return YELLOW_HEALTH;
-      case RED:
-        return RED_HEALTH;
-      default:
-        throw new IllegalArgumentException("Unsupported Elasticsearch status " + esStatus);
+    try {
+      ClusterHealthStatus esStatus = esClient.prepareClusterStats().get().getStatus();
+      switch (esStatus) {
+        case GREEN:
+          return Health.GREEN;
+        case YELLOW:
+          return YELLOW_HEALTH;
+        case RED:
+          return RED_HEALTH;
+        default:
+          throw new IllegalArgumentException("Unsupported Elasticsearch status " + esStatus);
+      }
+    } catch (Exception e) {
+      LOG.error("Failed to query ES status", e);
+      return RED_HEALTH_EXCEPTION_OCCURED;
     }
   }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java b/server/sonar-server/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java
index d24dce584cd..afdfd9df742 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java
@@ -25,24 +25,47 @@ import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import org.junit.Rule;
 import org.junit.Test;
+import org.mockito.Mockito;
 import org.sonar.cluster.health.NodeDetails;
 import org.sonar.cluster.health.NodeHealth;
+import org.sonar.server.es.EsClient;
 import org.sonar.server.es.EsTester;
 
 import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
 
 public class EsStatusClusterCheckTest {
 
   @Rule
   public EsTester esTester = new EsTester();
 
+  private final Random random = new Random();
   private EsStatusClusterCheck underTest = new EsStatusClusterCheck(esTester.client());
 
   @Test
+  public void check_ignores_NodeHealth_arg_and_returns_RED_with_cause_if_an_exception_occurs_checking_ES_cluster_status() {
+    Set<NodeHealth> nodeHealths = randomNodeHealths();
+    EsClient esClient = Mockito.mock(EsClient.class);
+    when(esClient.prepareClusterStats()).thenThrow(new RuntimeException("Faking an exception occuring while using the EsClient"));
+
+    Health health = new EsStatusClusterCheck(esClient).check(nodeHealths);
+
+    assertThat(health.getStatus()).isEqualTo(Health.Status.RED);
+    assertThat(health.getCauses()).containsOnly("Elasticsearch status is RED (unavailable)");
+  }
+
+  @Test
   public void check_ignores_NodeHealth_arg_and_returns_GREEN_without_cause_if_ES_cluster_status_is_GREEN() {
-    Random random = new Random();
-    Set<NodeHealth> nodeHealths = IntStream.range(0, random.nextInt(20))
+    Set<NodeHealth> nodeHealths = randomNodeHealths();
+
+    Health health = underTest.check(nodeHealths);
+
+    assertThat(health).isEqualTo(Health.GREEN);
+  }
+
+  private Set<NodeHealth> randomNodeHealths() {
+    return IntStream.range(0, random.nextInt(20))
       .mapToObj(i -> NodeHealth.newNodeHealthBuilder()
         .setStatus(NodeHealth.Status.values()[random.nextInt(NodeHealth.Status.values().length)])
         .setDetails(NodeDetails.newNodeDetailsBuilder()
@@ -54,9 +77,6 @@ public class EsStatusClusterCheckTest {
           .build())
         .build())
       .collect(Collectors.toSet());
-    Health health = underTest.check(nodeHealths);
-
-    assertThat(health).isEqualTo(Health.GREEN);
   }
 
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java b/server/sonar-server/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java
index 465ea1427b3..2530be657db 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java
@@ -21,9 +21,12 @@ package org.sonar.server.health;
 
 import org.junit.Rule;
 import org.junit.Test;
+import org.mockito.Mockito;
+import org.sonar.server.es.EsClient;
 import org.sonar.server.es.EsTester;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
 
 public class EsStatusNodeCheckTest {
 
@@ -33,6 +36,17 @@ public class EsStatusNodeCheckTest {
   private EsStatusNodeCheck underTest = new EsStatusNodeCheck(esTester.client());
 
   @Test
+  public void check_ignores_NodeHealth_arg_and_returns_RED_with_cause_if_an_exception_occurs_checking_ES_cluster_status() {
+    EsClient esClient = Mockito.mock(EsClient.class);
+    when(esClient.prepareClusterStats()).thenThrow(new RuntimeException("Faking an exception occuring while using the EsClient"));
+
+    Health health = new EsStatusNodeCheck(esClient).check();
+
+    assertThat(health.getStatus()).isEqualTo(Health.Status.RED);
+    assertThat(health.getCauses()).containsOnly("Elasticsearch status is RED (unavailable)");
+  }
+
+  @Test
   public void check_returns_GREEN_without_cause_if_ES_cluster_status_is_GREEN() {
     Health health = underTest.check();