From 62faa1de1e8a53424f210ab96e76549d6ad749b7 Mon Sep 17 00:00:00 2001 From: Zipeng WU Date: Thu, 2 Jun 2022 14:22:41 +0200 Subject: [PATCH] SONAR-16250 Global Health Status should be yellow if there are less than 3 search nodes --- .../sonar/server/health/EsStatusCheck.java | 38 +++++++++++-------- .../server/health/EsStatusClusterCheck.java | 22 ++++++++++- .../server/health/EsStatusNodeCheck.java | 4 +- .../health/EsStatusClusterCheckTest.java | 27 +++++++++++++ 4 files changed, 73 insertions(+), 18 deletions(-) diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java index 3abb51837a5..28eae7cfb4e 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java @@ -20,6 +20,7 @@ package org.sonar.server.health; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -38,7 +39,7 @@ abstract class EsStatusCheck { .setStatus(Health.Status.RED) .addCause("Elasticsearch status is RED") .build(); - private static final Health RED_HEALTH_UNAVAILABLE = newHealthCheckBuilder() + protected static final Health RED_HEALTH_UNAVAILABLE = newHealthCheckBuilder() .setStatus(Health.Status.RED) .addCause("Elasticsearch status is RED (unavailable)") .build(); @@ -49,25 +50,30 @@ abstract class EsStatusCheck { this.esClient = esClient; } - Health checkEsStatus() { + protected ClusterHealthResponse getEsClusterHealth() { try { - ClusterHealthStatus esStatus = esClient.clusterHealth(new ClusterHealthRequest()).getStatus(); - if (esStatus == null) { - return RED_HEALTH_UNAVAILABLE; - } - switch (esStatus) { - case GREEN: - return Health.GREEN; - case YELLOW: - return YELLOW_HEALTH; - case RED: - return RED_HEALTH; - default: - throw new IllegalArgumentException("Unsupported Elasticsearch status " + esStatus); - } + return esClient.clusterHealth(new ClusterHealthRequest()); } catch (Exception e) { LOG.error("Failed to query ES status", e); + return null; + } + } + + protected static Health extractStatusHealth(ClusterHealthResponse healthResponse) { + ClusterHealthStatus esStatus = healthResponse.getStatus(); + if (esStatus == null) { return RED_HEALTH_UNAVAILABLE; } + switch (esStatus) { + case GREEN: + return Health.GREEN; + case YELLOW: + return YELLOW_HEALTH; + case RED: + return RED_HEALTH; + default: + throw new IllegalArgumentException("Unsupported Elasticsearch status " + esStatus); + } } + } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusClusterCheck.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusClusterCheck.java index 5cfc998c700..a0028c655b8 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusClusterCheck.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusClusterCheck.java @@ -20,10 +20,13 @@ package org.sonar.server.health; import java.util.Set; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.sonar.process.cluster.health.NodeHealth; import org.sonar.server.es.EsClient; public class EsStatusClusterCheck extends EsStatusCheck implements ClusterHealthCheck { + private static final String MINIMUM_NODE_MESSAGE = "There should be at least three search nodes"; + private static final int RECOMMENDED_MIN_NUMBER_OF_ES_NODES = 3; public EsStatusClusterCheck(EsClient esClient) { super(esClient); @@ -31,7 +34,24 @@ public class EsStatusClusterCheck extends EsStatusCheck implements ClusterHealth @Override public Health check(Set nodeHealths) { - return checkEsStatus(); + ClusterHealthResponse esClusterHealth = this.getEsClusterHealth(); + if (esClusterHealth != null) { + Health minimumNodes = checkMinimumNodes(esClusterHealth); + Health clusterStatus = extractStatusHealth(esClusterHealth); + return HealthReducer.INSTANCE.apply(minimumNodes, clusterStatus); + } + return RED_HEALTH_UNAVAILABLE; + } + + private static Health checkMinimumNodes(ClusterHealthResponse esClusterHealth) { + int nodeCount = esClusterHealth.getNumberOfNodes(); + if (nodeCount < RECOMMENDED_MIN_NUMBER_OF_ES_NODES) { + return Health.newHealthCheckBuilder() + .setStatus(Health.Status.YELLOW) + .addCause(MINIMUM_NODE_MESSAGE) + .build(); + } + return Health.GREEN; } } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusNodeCheck.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusNodeCheck.java index 955cb2d4cfc..438e33fa569 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusNodeCheck.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusNodeCheck.java @@ -19,6 +19,7 @@ */ package org.sonar.server.health; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.sonar.server.es.EsClient; /** @@ -32,6 +33,7 @@ public class EsStatusNodeCheck extends EsStatusCheck implements NodeHealthCheck @Override public Health check() { - return super.checkEsStatus(); + ClusterHealthResponse healthResponse = getEsClusterHealth(); + return healthResponse != null ? extractStatusHealth(healthResponse) : RED_HEALTH_UNAVAILABLE; } } diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java index bc959def3ce..fb99c2b1f22 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java @@ -34,6 +34,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.when; +import static org.sonar.process.cluster.health.NodeHealth.Status.GREEN; public class EsStatusClusterCheckTest { @@ -56,12 +57,38 @@ public class EsStatusClusterCheckTest { public void check_ignores_NodeHealth_arg_and_returns_GREEN_without_cause_if_ES_cluster_status_is_GREEN() { Set nodeHealths = ImmutableSet.of(newNodeHealth(NodeHealth.Status.YELLOW)); when(esClient.clusterHealth(any()).getStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esClient.clusterHealth(any()).getNumberOfNodes()).thenReturn(3); Health health = underTest.check(nodeHealths); assertThat(health).isEqualTo(Health.GREEN); } + @Test + public void check_returns_YELLOW_with_cause_if_ES_cluster_has_less_than_three_nodes_but_status_is_green() { + Set nodeHealths = ImmutableSet.of(newNodeHealth(GREEN)); + when(esClient.clusterHealth(any()).getStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esClient.clusterHealth(any()).getNumberOfNodes()).thenReturn(2); + + Health health = underTest.check(nodeHealths); + + assertThat(health.getStatus()).isEqualTo(Health.Status.YELLOW); + assertThat(health.getCauses()).containsOnly("There should be at least three search nodes"); + } + + @Test + public void check_returns_RED_with_cause_if_ES_cluster_has_less_than_three_nodes_and_status_is_RED() { + Set nodeHealths = ImmutableSet.of(newNodeHealth(GREEN)); + when(esClient.clusterHealth(any()).getStatus()).thenReturn(ClusterHealthStatus.RED); + when(esClient.clusterHealth(any()).getNumberOfNodes()).thenReturn(2); + + Health health = underTest.check(nodeHealths); + + assertThat(health.getStatus()).isEqualTo(Health.Status.RED); + assertThat(health.getCauses()).contains("Elasticsearch status is RED", "There should be at least three search nodes"); + } + + private NodeHealth newNodeHealth(NodeHealth.Status status) { return NodeHealth.newNodeHealthBuilder() .setStatus(status) -- 2.39.5