SONAR-9741 check search nodes health in api/system/health

8 files changed, 815 insertions, 75 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/health/AppNodeClusterCheck.java b/server/sonar-server/src/main/java/org/sonar/server/health/AppNodeClusterCheck.java
index f91b51ab5f4..b3c09941604 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/health/AppNodeClusterCheck.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/health/AppNodeClusterCheck.java
@@ -38,12 +38,12 @@ public class AppNodeClusterCheck implements ClusterHealthCheck {
       .filter(s -> s.getDetails().getType() == NodeDetails.Type.APPLICATION)
       .collect(toSet());
 
-    return Arrays.stream(AppNodeClusterHealthChecks.values())
+    return Arrays.stream(AppNodeClusterHealthSubChecks.values())
       .map(s -> s.check(appNodes))
       .reduce(Health.GREEN, HealthReducer.INSTANCE);
   }
 
-  private enum AppNodeClusterHealthChecks implements ClusterHealthCheck {
+  private enum AppNodeClusterHealthSubChecks implements ClusterHealthSubCheck {
     NO_APPLICATION_NODE() {
       @Override
       public Health check(Set<NodeHealth> appNodes) {
@@ -79,8 +79,8 @@ public class AppNodeClusterCheck implements ClusterHealthCheck {
           return Health.GREEN;
         }
 
-        long redNodesCount = appNodes.stream().filter(s -> s.getStatus() == RED).count();
-        long yellowNodesCount = appNodes.stream().filter(s -> s.getStatus() == YELLOW).count();
+        long redNodesCount = withStatus(appNodes, RED).count();
+        long yellowNodesCount = withStatus(appNodes, YELLOW).count();
         if (redNodesCount == 0 && yellowNodesCount == 0) {
           return Health.GREEN;
         }
@@ -102,7 +102,7 @@ public class AppNodeClusterCheck implements ClusterHealthCheck {
         } else if (yellowNodesCount > 0) {
           builder.addCause("At least one application node is YELLOW");
         }
-        long greenNodesCount = appNodes.stream().filter(s -> s.getStatus() == GREEN).count();
+        long greenNodesCount = withStatus(appNodes, GREEN).count();
         builder.setStatus(greenNodesCount > 0 || yellowNodesCount > 0 ? Health.Status.YELLOW : Health.Status.RED);
 
         return builder.build();
diff --git a/server/sonar-server/src/main/java/org/sonar/server/health/ClusterHealthSubCheck.java b/server/sonar-server/src/main/java/org/sonar/server/health/ClusterHealthSubCheck.java
new file mode 100644
index 00000000000..b44731adaa3
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/health/ClusterHealthSubCheck.java
@@ -0,0 +1,39 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.health;
+
+import java.util.Set;
+import java.util.stream.Stream;
+import org.sonar.cluster.health.NodeHealth;
+
+interface ClusterHealthSubCheck extends ClusterHealthCheck {
+
+  default Stream<NodeHealth> withStatus(Set<NodeHealth> searchNodes, NodeHealth.Status... statuses) {
+    return searchNodes.stream()
+        .filter(t -> {
+          for (NodeHealth.Status status : statuses) {
+            if (status == t.getStatus()) {
+              return true;
+            }
+          }
+          return false;
+        });
+  }
+}
diff --git a/server/sonar-server/src/main/java/org/sonar/server/health/SearchNodeClusterCheck.java b/server/sonar-server/src/main/java/org/sonar/server/health/SearchNodeClusterCheck.java
new file mode 100644
index 00000000000..0f1ede28481
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/health/SearchNodeClusterCheck.java
@@ -0,0 +1,129 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.health;
+
+import java.util.Arrays;
+import java.util.Set;
+import org.sonar.cluster.health.NodeDetails;
+import org.sonar.cluster.health.NodeHealth;
+
+import static org.sonar.cluster.health.NodeHealth.Status.GREEN;
+import static org.sonar.cluster.health.NodeHealth.Status.RED;
+import static org.sonar.cluster.health.NodeHealth.Status.YELLOW;
+import static org.sonar.core.util.stream.MoreCollectors.toSet;
+import static org.sonar.server.health.Health.newHealthCheckBuilder;
+
+public class SearchNodeClusterCheck implements ClusterHealthCheck {
+
+  @Override
+  public Health check(Set<NodeHealth> nodeHealths) {
+    Set<NodeHealth> searchNOdes = nodeHealths.stream()
+      .filter(s -> s.getDetails().getType() == NodeDetails.Type.SEARCH)
+      .collect(toSet());
+
+    return Arrays.stream(SearchNodeClusterSubChecks.values())
+      .map(s -> s.check(searchNOdes))
+      .reduce(Health.GREEN, HealthReducer.INSTANCE);
+  }
+
+  private enum SearchNodeClusterSubChecks implements ClusterHealthSubCheck {
+    NO_SEARCH_NODE() {
+      @Override
+      public Health check(Set<NodeHealth> searchNodes) {
+        int searchNodeCount = searchNodes.size();
+        if (searchNodeCount == 0) {
+          return newHealthCheckBuilder()
+            .setStatus(Health.Status.RED)
+            .addCause("No search node")
+            .build();
+        }
+        return Health.GREEN;
+      }
+    },
+    NUMBER_OF_NODES() {
+      @Override
+      public Health check(Set<NodeHealth> searchNodes) {
+        int searchNodeCount = searchNodes.size();
+        if (searchNodeCount == 0) {
+          // skipping this check
+          return Health.GREEN;
+        }
+
+        if (searchNodeCount < 3) {
+          long yellowGreenNodesCount = withStatus(searchNodes, GREEN, YELLOW).count();
+          return newHealthCheckBuilder()
+            .setStatus(yellowGreenNodesCount > 1 ? Health.Status.YELLOW : Health.Status.RED)
+            .addCause("There should be at least three search nodes")
+            .build();
+        }
+        if (searchNodeCount > 3 && isEven(searchNodeCount)) {
+          return newHealthCheckBuilder()
+            .setStatus(Health.Status.YELLOW)
+            .addCause("There should be an odd number of search nodes")
+            .build();
+        }
+        return Health.GREEN;
+      }
+
+      private boolean isEven(int searchNodeCount) {
+        return searchNodeCount % 2 == 0;
+      }
+    },
+    RED_OR_YELLOW_NODES() {
+      @Override
+      public Health check(Set<NodeHealth> searchNodes) {
+        int searchNodeCount = searchNodes.size();
+        if (searchNodeCount == 0) {
+          // skipping this check
+          return Health.GREEN;
+        }
+
+        long redNodesCount = withStatus(searchNodes, RED).count();
+        long yellowNodesCount = withStatus(searchNodes, YELLOW).count();
+        if (redNodesCount == 0 && yellowNodesCount == 0) {
+          return Health.GREEN;
+        }
+
+        Health.Builder builder = newHealthCheckBuilder();
+        if (redNodesCount == searchNodeCount) {
+          return builder
+            .setStatus(Health.Status.RED)
+            .addCause("Status of all search nodes is RED")
+            .build();
+        } else if (redNodesCount > 0) {
+          builder.addCause("At least one search node is RED");
+        }
+        if (yellowNodesCount == searchNodeCount) {
+          return builder
+            .setStatus(Health.Status.YELLOW)
+            .addCause("Status of all search nodes is YELLOW")
+            .build();
+        } else if (yellowNodesCount > 0) {
+          builder.addCause("At least one search node is YELLOW");
+        }
+
+        long greenNodesCount = withStatus(searchNodes, GREEN).count();
+        builder.setStatus(greenNodesCount + yellowNodesCount > 1 ? Health.Status.YELLOW : Health.Status.RED);
+
+        return builder.build();
+      }
+    }
+  }
+}
diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ws/HealthActionModule.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/HealthActionModule.java
index 0985a9dda9c..05cf290cedc 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/platform/ws/HealthActionModule.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/HealthActionModule.java
@@ -26,6 +26,7 @@ import org.sonar.server.health.DbConnectionNodeCheck;
 import org.sonar.server.health.EsStatusClusterCheck;
 import org.sonar.server.health.EsStatusNodeCheck;
 import org.sonar.server.health.HealthCheckerImpl;
+import org.sonar.server.health.SearchNodeClusterCheck;
 import org.sonar.server.health.WebServerStatusNodeCheck;
 import org.sonar.server.platform.WebServer;
 
@@ -46,7 +47,8 @@ public class HealthActionModule extends Module {
     if (!webServer.isStandalone()) {
       // ClusterHealthCheck implementations
       add(EsStatusClusterCheck.class,
-        AppNodeClusterCheck.class);
+        AppNodeClusterCheck.class,
+        SearchNodeClusterCheck.class);
     }
 
     add(HealthCheckerImpl.class,
diff --git a/server/sonar-server/src/test/java/org/sonar/server/health/AppNodeClusterCheckTest.java b/server/sonar-server/src/test/java/org/sonar/server/health/AppNodeClusterCheckTest.java
index 02b627f59e9..a138a705c50 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/health/AppNodeClusterCheckTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/health/AppNodeClusterCheckTest.java
@@ -19,17 +19,11 @@
  */
 package org.sonar.server.health;
 
-import com.google.common.collect.ImmutableList;
 import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Objects;
 import java.util.Random;
 import java.util.Set;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
-import javax.annotation.Nullable;
-import org.assertj.core.api.AbstractAssert;
 import org.junit.Test;
 import org.sonar.cluster.health.NodeDetails;
 import org.sonar.cluster.health.NodeHealth;
@@ -40,6 +34,7 @@ import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
 import static org.sonar.cluster.health.NodeHealth.Status.GREEN;
 import static org.sonar.cluster.health.NodeHealth.Status.RED;
 import static org.sonar.cluster.health.NodeHealth.Status.YELLOW;
+import static org.sonar.server.health.HealthAssert.assertThat;
 
 public class AppNodeClusterCheckTest {
   private final Random random = new Random();
@@ -288,66 +283,4 @@ public class AppNodeClusterCheckTest {
       .build();
   }
 
-  public static HealthAssert assertThat(Health actual) {
-    return new HealthAssert(actual);
-  }
-
-  private static final class HealthAssert extends AbstractAssert<HealthAssert, Health> {
-    private Set<NodeHealth> nodeHealths;
-
-    protected HealthAssert(Health actual) {
-      super(actual, HealthAssert.class);
-    }
-
-    public HealthAssert forInput(Set<NodeHealth> nodeHealths) {
-      this.nodeHealths = nodeHealths;
-
-      return this;
-    }
-
-    public HealthAssert hasStatus(Health.Status expected) {
-      isNotNull();
-
-      if (actual.getStatus() != expected) {
-        failWithMessage(
-          "Expected Status of Health to be <%s> but was <%s> for NodeHealth \n%s",
-          expected,
-          actual.getStatus(),
-          printStatusesAndTypes(this.nodeHealths));
-      }
-
-      return this;
-    }
-
-    public HealthAssert andCauses(String... causes) {
-      isNotNull();
-
-      if (!checkCauses(causes)) {
-        failWithMessage(
-          "Expected causes of Health to contain only \n%s\n but was \n%s\n for NodeHealth \n%s",
-          Arrays.asList(causes),
-          actual.getCauses(),
-          printStatusesAndTypes(this.nodeHealths));
-      }
-
-      return this;
-    }
-
-    private String printStatusesAndTypes(@Nullable Set<NodeHealth> nodeHealths) {
-      if (nodeHealths == null) {
-        return "<null>";
-      }
-      return nodeHealths.stream()
-        .map(s -> ImmutableList.of(s.getDetails().getType().name(), s.getStatus().name()))
-        .map(String::valueOf)
-        .collect(Collectors.joining(","));
-    }
-
-    private boolean checkCauses(String... causes) {
-      if (causes.length != this.actual.getCauses().size()) {
-        return false;
-      }
-      return Objects.equals(new HashSet<>(Arrays.asList(causes)), this.actual.getCauses());
-    }
-  }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/health/HealthAssert.java b/server/sonar-server/src/test/java/org/sonar/server/health/HealthAssert.java
new file mode 100644
index 00000000000..4ea8c4f80ac
--- /dev/null
+++ b/server/sonar-server/src/test/java/org/sonar/server/health/HealthAssert.java
@@ -0,0 +1,97 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.health;
+
+import com.google.common.collect.ImmutableList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.assertj.core.api.AbstractAssert;
+import org.sonar.cluster.health.NodeHealth;
+
+final class HealthAssert extends AbstractAssert<HealthAssert, Health> {
+  private Set<NodeHealth> nodeHealths;
+
+  private HealthAssert(Health actual) {
+    super(actual, HealthAssert.class);
+  }
+
+  public static HealthAssert assertThat(Health actual) {
+    return new HealthAssert(actual);
+  }
+
+  public HealthAssert forInput(Set<NodeHealth> nodeHealths) {
+    this.nodeHealths = nodeHealths;
+
+    return this;
+  }
+
+  public HealthAssert hasStatus(Health.Status expected) {
+    isNotNull();
+
+    if (actual.getStatus() != expected) {
+      failWithMessage(
+        "Expected Status of Health to be <%s> but was <%s> for NodeHealth \n%s",
+        expected,
+        actual.getStatus(),
+        printStatusesAndTypes(this.nodeHealths));
+    }
+
+    return this;
+  }
+
+  public HealthAssert andCauses(String... causes) {
+    isNotNull();
+
+    if (!checkCauses(causes)) {
+      failWithMessage(
+        "Expected causes of Health to contain only \n%s\n but was \n%s\n for NodeHealth \n%s",
+        Arrays.asList(causes),
+        actual.getCauses(),
+        printStatusesAndTypes(this.nodeHealths));
+    }
+
+    return this;
+  }
+
+  private String printStatusesAndTypes(@Nullable Set<NodeHealth> nodeHealths) {
+    if (nodeHealths == null) {
+      return "<null>";
+    }
+    return nodeHealths.stream()
+      // sort by type then status for debugging convenience
+      .sorted(Comparator.<NodeHealth>comparingInt(s1 -> s1.getDetails().getType().ordinal())
+        .thenComparingInt(s -> s.getStatus().ordinal()))
+      .map(s -> ImmutableList.of(s.getDetails().getType().name(), s.getStatus().name()))
+      .map(String::valueOf)
+      .collect(Collectors.joining(","));
+  }
+
+  private boolean checkCauses(String... causes) {
+    if (causes.length != this.actual.getCauses().size()) {
+      return false;
+    }
+    return Objects.equals(new HashSet<>(Arrays.asList(causes)), this.actual.getCauses());
+  }
+}
diff --git a/server/sonar-server/src/test/java/org/sonar/server/health/SearchNodeClusterCheckTest.java b/server/sonar-server/src/test/java/org/sonar/server/health/SearchNodeClusterCheckTest.java
new file mode 100644
index 00000000000..67eadaf503a
--- /dev/null
+++ b/server/sonar-server/src/test/java/org/sonar/server/health/SearchNodeClusterCheckTest.java
@@ -0,0 +1,538 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.health;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import org.junit.Test;
+import org.sonar.cluster.health.NodeDetails;
+import org.sonar.cluster.health.NodeHealth;
+
+import static java.util.stream.Collectors.toSet;
+import static java.util.stream.Stream.of;
+import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
+import static org.sonar.cluster.health.NodeHealth.Status.GREEN;
+import static org.sonar.cluster.health.NodeHealth.Status.RED;
+import static org.sonar.cluster.health.NodeHealth.Status.YELLOW;
+import static org.sonar.server.health.HealthAssert.assertThat;
+
+public class SearchNodeClusterCheckTest {
+  private final Random random = new Random();
+
+  private SearchNodeClusterCheck underTest = new SearchNodeClusterCheck();
+
+  @Test
+  public void status_RED_when_no_search_node() {
+    Set<NodeHealth> nodeHealths = nodeHealths().collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.RED)
+      .andCauses("No search node");
+  }
+
+  @Test
+  public void status_RED_when_single_RED_search_node() {
+    Set<NodeHealth> nodeHealths = nodeHealths(RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.RED)
+      .andCauses("Status of all search nodes is RED",
+        "There should be at least three search nodes");
+  }
+
+  @Test
+  public void status_RED_when_single_YELLOW_search_node() {
+    Set<NodeHealth> nodeHealths = nodeHealths(YELLOW).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.RED)
+      .andCauses("Status of all search nodes is YELLOW",
+        "There should be at least three search nodes");
+  }
+
+  @Test
+  public void status_RED_when_single_GREEN_search_node() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.RED)
+      .andCauses("There should be at least three search nodes");
+  }
+
+  @Test
+  public void status_RED_when_two_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(RED, RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.RED)
+      .andCauses("Status of all search nodes is RED",
+        "There should be at least three search nodes");
+  }
+
+  @Test
+  public void status_YELLOW_when_two_YELLOW_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(YELLOW, YELLOW).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("Status of all search nodes is YELLOW",
+        "There should be at least three search nodes");
+  }
+
+  @Test
+  public void status_YELLOW_when_two_GREEN_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, GREEN).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be at least three search nodes");
+  }
+
+  @Test
+  public void status_YELLOW_when_one_GREEN_and_one_YELLOW_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, YELLOW).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("At least one search node is YELLOW",
+        "There should be at least three search nodes");
+  }
+
+  @Test
+  public void status_RED_when_one_GREEN_and_one_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.RED)
+      .andCauses("At least one search node is RED",
+        "There should be at least three search nodes");
+  }
+
+  @Test
+  public void status_RED_when_one_YELLOW_and_one_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(YELLOW, RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.RED)
+      .andCauses("At least one search node is RED",
+        "At least one search node is YELLOW",
+        "There should be at least three search nodes");
+  }
+
+  @Test
+  public void status_RED_when_three_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(RED, RED, RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.RED)
+      .andCauses("Status of all search nodes is RED");
+  }
+
+  @Test
+  public void status_YELLOW_when_three_YELLOW_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(YELLOW, YELLOW, YELLOW).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("Status of all search nodes is YELLOW");
+  }
+
+  @Test
+  public void status_GREEN_when_three_GREEN_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, GREEN, GREEN).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.GREEN)
+      .andCauses();
+  }
+
+  @Test
+  public void status_RED_when_two_RED_and_one_YELLOW_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(RED, RED, YELLOW).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.RED)
+      .andCauses("At least one search node is RED",
+        "At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_two_YELLOW_and_one_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(RED, YELLOW, YELLOW).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("At least one search node is RED",
+        "At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_two_GREEN_and_one_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, GREEN, RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("At least one search node is RED");
+  }
+
+  @Test
+  public void status_YELLOW_when_two_GREEN_and_one_YELLOW_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, GREEN, YELLOW).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_two_GREEN_and_two_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, GREEN, RED, RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes",
+        "At least one search node is RED");
+  }
+
+  @Test
+  public void status_YELLOW_when_two_GREEN_and_two_YELLOW_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, GREEN, YELLOW, YELLOW).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes",
+        "At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_two_GREEN_and_one_YELLOW_and_one_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, GREEN, YELLOW, RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes",
+        "At least one search node is RED",
+        "At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_one_GREEN_one_YELLOW_and_two_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(GREEN, YELLOW, RED, RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes",
+        "At least one search node is RED",
+        "At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_three_YELLOW_and_one_GREEN_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(YELLOW, YELLOW, YELLOW, GREEN).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes",
+        "At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_three_YELLOW_and_one_RED_search_nodes() {
+    Set<NodeHealth> nodeHealths = nodeHealths(YELLOW, YELLOW, YELLOW, RED).collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes",
+        "At least one search node is RED",
+        "At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_GREEN_when_three_GREEN_search_nodes_and_even_number_of_GREEN() {
+    Set<NodeHealth> nodeHealths = of(
+      // 0, 2, 4 or 6 GREEN
+      evenNumberOfAppNodeHealthOfAnyStatus(GREEN),
+      // 3 GREEN
+      nodeHealths(GREEN, GREEN, GREEN))
+        .flatMap(s -> s)
+        .collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.GREEN)
+      .andCauses();
+  }
+
+  @Test
+  public void status_YELLOW_when_three_GREEN_search_nodes_and_even_number_of_GREEN_and_YELLOW() {
+    Set<NodeHealth> nodeHealths = of(
+      of(searchNodeHealth(GREEN), searchNodeHealth(YELLOW)),
+      // 0, 2, 4 or 6 GREEN
+      evenNumberOfAppNodeHealthOfAnyStatus(GREEN),
+      // 3 GREEN
+      nodeHealths(GREEN, GREEN, GREEN))
+        .flatMap(s -> s)
+        .collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_three_GREEN_search_nodes_and_even_number_of_GREEN_and_RED() {
+    Set<NodeHealth> nodeHealths = of(
+      // at least one GREEN and one RED
+      of(searchNodeHealth(GREEN), searchNodeHealth(RED)),
+      // 0, 2, 4 or 6 GREEN or RED
+      evenNumberOfAppNodeHealthOfAnyStatus(GREEN, RED),
+      // 3 GREEN
+      nodeHealths(GREEN, GREEN, GREEN))
+        .flatMap(s -> s)
+        .collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("At least one search node is RED");
+  }
+
+  @Test
+  public void status_YELLOW_when_three_GREEN_search_nodes_and_even_number_of_YELLOW_and_RED() {
+    Set<NodeHealth> nodeHealths = of(
+      // at least one YELLOW and one RED
+      of(searchNodeHealth(YELLOW), searchNodeHealth(RED)),
+      // 0, 2, 4 or 6 GREEN, YELLOW or RED
+      evenNumberOfAppNodeHealthOfAnyStatus(YELLOW, RED, GREEN),
+      // 3 GREEN
+      nodeHealths(GREEN, GREEN, GREEN))
+        .flatMap(s -> s)
+        .collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("At least one search node is RED",
+        "At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_three_GREEN_search_nodes_and_odd_number_of_GREEN() {
+    Set<NodeHealth> nodeHealths = of(
+      // 1, 3, 5, or 7 GREEN
+      oddNumberOfAppNodeHealthOfAnyStatus(GREEN),
+      // 3 GREEN
+      nodeHealths(GREEN, GREEN, GREEN))
+        .flatMap(s -> s)
+        .collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes");
+  }
+
+  @Test
+  public void status_YELLOW_when_three_GREEN_search_nodes_and_odd_number_of_YELLOW() {
+    Set<NodeHealth> nodeHealths = of(
+      // 1, 3, 5, or 7 YELLOW
+      oddNumberOfAppNodeHealthOfAnyStatus(YELLOW),
+      // 3 GREEN
+      nodeHealths(GREEN, GREEN, GREEN))
+        .flatMap(s -> s)
+        .collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes",
+        "At least one search node is YELLOW");
+  }
+
+  @Test
+  public void status_YELLOW_when_three_GREEN_search_nodes_and_odd_number_of_RED() {
+    Set<NodeHealth> nodeHealths = of(
+      // 1, 3, 5, or 7 RED
+      oddNumberOfAppNodeHealthOfAnyStatus(RED),
+      // 3 GREEN
+      nodeHealths(GREEN, GREEN, GREEN))
+        .flatMap(s -> s)
+        .collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes",
+        "At least one search node is RED");
+  }
+
+  @Test
+  public void status_YELLOW_when_three_GREEN_search_nodes_and_odd_number_of_RED_and_YELLOW() {
+    Set<NodeHealth> nodeHealths = of(
+      // at least one YELLOW and one RED
+      of(searchNodeHealth(YELLOW), searchNodeHealth(RED)),
+      // 1, 3, 5, or 7 GREEN RED or YELLOW
+      oddNumberOfAppNodeHealthOfAnyStatus(RED, YELLOW, GREEN),
+      // 3 GREEN
+      nodeHealths(GREEN, GREEN, GREEN))
+        .flatMap(s -> s)
+        .collect(toSet());
+
+    Health check = underTest.check(nodeHealths);
+
+    assertThat(check)
+      .forInput(nodeHealths)
+      .hasStatus(Health.Status.YELLOW)
+      .andCauses("There should be an odd number of search nodes",
+        "At least one search node is RED",
+        "At least one search node is YELLOW");
+  }
+
+  private Stream<NodeHealth> nodeHealths(NodeHealth.Status... searchNodeStatuses) {
+    return of(
+      // random number of Application nodes with random status
+      IntStream.range(0, random.nextInt(3))
+        .mapToObj(i -> nodeHealth(NodeDetails.Type.APPLICATION, NodeHealth.Status.values()[random.nextInt(NodeHealth.Status.values().length)])),
+      Arrays.stream(searchNodeStatuses).map(this::searchNodeHealth))
+        .flatMap(s -> s);
+  }
+
+  private NodeHealth searchNodeHealth(NodeHealth.Status status) {
+    return nodeHealth(NodeDetails.Type.SEARCH, status);
+  }
+
+  private NodeHealth nodeHealth(NodeDetails.Type type, NodeHealth.Status status) {
+    return NodeHealth.newNodeHealthBuilder()
+      .setStatus(status)
+      .setDetails(NodeDetails.newNodeDetailsBuilder()
+        .setType(type)
+        .setHost(randomAlphanumeric(32))
+        .setName(randomAlphanumeric(32))
+        .setPort(1 + random.nextInt(88))
+        .setStarted(1 + random.nextInt(54))
+        .build())
+      .setDate(1 + random.nextInt(2323))
+      .build();
+  }
+
+  /**
+   * Between 0, 2, 4 or 6 NodeHealth of Application node with any of the specified statuses.
+   */
+  private Stream<NodeHealth> evenNumberOfAppNodeHealthOfAnyStatus(NodeHealth.Status... randomStatuses) {
+    return IntStream.range(0, 2 * random.nextInt(3))
+      .mapToObj(i -> searchNodeHealth(randomStatuses[random.nextInt(randomStatuses.length)]));
+  }
+
+  /**
+   * Between 1, 3, 5, or 7 NodeHealth of Application node with any of the specified statuses.
+   */
+  private Stream<NodeHealth> oddNumberOfAppNodeHealthOfAnyStatus(NodeHealth.Status... randomStatuses) {
+    return IntStream.range(0, 2 * random.nextInt(3) + 1)
+      .mapToObj(i -> searchNodeHealth(randomStatuses[random.nextInt(randomStatuses.length)]));
+  }
+
+}
diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/ws/HealthActionModuleTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/ws/HealthActionModuleTest.java
index e4ac3a8448e..07d1a027699 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/platform/ws/HealthActionModuleTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/platform/ws/HealthActionModuleTest.java
@@ -34,6 +34,7 @@ import org.sonar.server.health.EsStatusClusterCheck;
 import org.sonar.server.health.EsStatusNodeCheck;
 import org.sonar.server.health.HealthCheckerImpl;
 import org.sonar.server.health.NodeHealthCheck;
+import org.sonar.server.health.SearchNodeClusterCheck;
 import org.sonar.server.health.WebServerStatusNodeCheck;
 import org.sonar.server.platform.WebServer;
 
@@ -98,8 +99,9 @@ public class HealthActionModuleTest {
 
     List<Class<?>> checks = classesAddedToContainer(container).stream().filter(ClusterHealthCheck.class::isAssignableFrom).collect(Collectors.toList());
     assertThat(checks)
-      .hasSize(2)
+      .hasSize(3)
       .contains(EsStatusClusterCheck.class)
+      .contains(SearchNodeClusterCheck.class)
       .contains(AppNodeClusterCheck.class);
   }