SONAR-9712 reduce types of cluster nodes to only: application OR search

4 files changed, 129 insertions, 55 deletions

diff --git a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java
index b6b8ce39128..3861936d858 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java
@@ -26,23 +26,27 @@ import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.net.SocketException;
 import java.net.UnknownHostException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.process.MessageException;
+import org.sonar.process.NodeType;
 import org.sonar.process.ProcessId;
 import org.sonar.process.ProcessProperties;
 import org.sonar.process.Props;
 
 import static com.google.common.net.InetAddresses.forString;
 import static java.lang.String.format;
+import static java.util.Arrays.asList;
 import static java.util.Arrays.stream;
+import static java.util.Collections.singletonList;
 import static org.apache.commons.lang.StringUtils.isBlank;
 import static org.sonar.process.ProcessProperties.CLUSTER_ENABLED;
 import static org.sonar.process.ProcessProperties.CLUSTER_HOSTS;
 import static org.sonar.process.ProcessProperties.CLUSTER_NETWORK_INTERFACES;
+import static org.sonar.process.ProcessProperties.CLUSTER_NODE_TYPE;
 import static org.sonar.process.ProcessProperties.CLUSTER_SEARCH_HOSTS;
 import static org.sonar.process.ProcessProperties.CLUSTER_WEB_LEADER;
 import static org.sonar.process.ProcessProperties.JDBC_URL;
@@ -65,16 +69,22 @@ public class ClusterSettings implements Consumer<Props> {
       throw new MessageException(format("Property [%s] is forbidden", CLUSTER_WEB_LEADER));
     }
 
-    if (props.valueAsBoolean(ProcessProperties.CLUSTER_ENABLED) &&
-      !props.valueAsBoolean(ProcessProperties.CLUSTER_SEARCH_DISABLED, false)
-      ) {
-      ensureMandatoryProperty(props, SEARCH_HOST);
-      ensureNotLoopback(props, SEARCH_HOST);
-    }
     // Mandatory properties
+    ensureMandatoryProperty(props, CLUSTER_NODE_TYPE);
     ensureMandatoryProperty(props, CLUSTER_HOSTS);
     ensureMandatoryProperty(props, CLUSTER_SEARCH_HOSTS);
 
+    String nodeTypeValue = props.value(ProcessProperties.CLUSTER_NODE_TYPE);
+    if (!NodeType.isValid(nodeTypeValue)) {
+      throw new MessageException(format("Invalid nodeTypeValue for [%s]: [%s], only [%s] are allowed", ProcessProperties.CLUSTER_NODE_TYPE, nodeTypeValue,
+        Arrays.stream(NodeType.values()).map(NodeType::getValue).collect(Collectors.joining(", "))));
+    }
+    NodeType nodeType = NodeType.parse(nodeTypeValue);
+    if (nodeType == NodeType.SEARCH) {
+      ensureMandatoryProperty(props, SEARCH_HOST);
+      ensureNotLoopback(props, SEARCH_HOST);
+    }
+
     // H2 Database is not allowed in cluster mode
     String jdbcUrl = props.value(JDBC_URL);
     if (isBlank(jdbcUrl) || jdbcUrl.startsWith("jdbc:h2:")) {
@@ -162,24 +172,27 @@ public class ClusterSettings implements Consumer<Props> {
 
   public static List<ProcessId> getEnabledProcesses(AppSettings settings) {
     if (!isClusterEnabled(settings)) {
-      return Arrays.asList(ProcessId.ELASTICSEARCH, ProcessId.WEB_SERVER, ProcessId.COMPUTE_ENGINE);
-    }
-    List<ProcessId> enabled = new ArrayList<>();
-    if (!settings.getProps().valueAsBoolean(ProcessProperties.CLUSTER_SEARCH_DISABLED)) {
-      enabled.add(ProcessId.ELASTICSEARCH);
+      return asList(ProcessId.ELASTICSEARCH, ProcessId.WEB_SERVER, ProcessId.COMPUTE_ENGINE);
     }
-    if (!settings.getProps().valueAsBoolean(ProcessProperties.CLUSTER_WEB_DISABLED)) {
-      enabled.add(ProcessId.WEB_SERVER);
-    }
-
-    if (!settings.getProps().valueAsBoolean(ProcessProperties.CLUSTER_CE_DISABLED)) {
-      enabled.add(ProcessId.COMPUTE_ENGINE);
+    NodeType nodeType = NodeType.parse(settings.getValue(ProcessProperties.CLUSTER_NODE_TYPE).orElse(null));
+    switch (nodeType) {
+      case APPLICATION:
+        return asList(ProcessId.WEB_SERVER, ProcessId.COMPUTE_ENGINE);
+      case SEARCH:
+        return singletonList(ProcessId.ELASTICSEARCH);
+      default:
+        throw new IllegalStateException("Unexpected node type "+nodeType);
     }
-    return enabled;
   }
 
   public static boolean isLocalElasticsearchEnabled(AppSettings settings) {
-    return !isClusterEnabled(settings.getProps()) ||
-      !settings.getProps().valueAsBoolean(ProcessProperties.CLUSTER_SEARCH_DISABLED);
+
+    // elasticsearch is enabled on "search" nodes, but disabled on "application" nodes
+    if (isClusterEnabled(settings.getProps())) {
+      return NodeType.parse(settings.getValue(ProcessProperties.CLUSTER_NODE_TYPE).orElse(null)) == NodeType.SEARCH;
+    }
+
+    // elasticsearch is enabled in standalone mode
+    return true;
   }
 }
diff --git a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java
index f7336820816..7d3c5cee24d 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java
@@ -94,7 +94,6 @@ public class SchedulerImplTest {
 
   @Test
   public void start_and_stop_sequence_of_ES_WEB_CE_in_order() throws Exception {
-    enableAllProcesses();
     SchedulerImpl underTest = newScheduler();
     underTest.schedule();
 
@@ -132,7 +131,6 @@ public class SchedulerImplTest {
   }
 
   private void enableAllProcesses() {
-    settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
   }
 
   @Test
@@ -152,7 +150,6 @@ public class SchedulerImplTest {
 
   @Test
   public void all_processes_are_stopped_if_one_process_fails_to_start() throws Exception {
-    enableAllProcesses();
     SchedulerImpl underTest = newScheduler();
     processLauncher.makeStartupFail = COMPUTE_ENGINE;
 
@@ -239,59 +236,87 @@ public class SchedulerImplTest {
   }
 
   @Test
-  public void web_follower_starts_only_when_web_leader_is_operational() throws Exception {
+  public void search_node_starts_only_elasticsearch() throws Exception {
+    settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search");
+    SchedulerImpl underTest = newScheduler();
+    underTest.schedule();
+
+    processLauncher.waitForProcessAlive(ProcessId.ELASTICSEARCH);
+    assertThat(processLauncher.processes).hasSize(1);
+
+    underTest.terminate();
+  }
+
+  @Test
+  public void application_node_starts_only_web_and_ce() throws Exception {
+    appState.setOperational(ProcessId.ELASTICSEARCH);
+    settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application");
+    SchedulerImpl underTest = newScheduler();
+    underTest.schedule();
+
+    TestProcess web = processLauncher.waitForProcessAlive(WEB_SERVER);
+    web.operational = true;
+    processLauncher.waitForProcessAlive(COMPUTE_ENGINE);
+    assertThat(processLauncher.processes).hasSize(2);
+
+    underTest.terminate();
+  }
+
+  @Test
+  public void search_node_starts_even_if_web_leader_is_not_yet_operational() throws Exception {
     // leader takes the lock, so underTest won't get it
     assertThat(appState.tryToLockWebLeader()).isTrue();
 
     appState.setOperational(ProcessId.ELASTICSEARCH);
-    enableAllProcesses();
+    settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search");
     SchedulerImpl underTest = newScheduler();
     underTest.schedule();
 
     processLauncher.waitForProcessAlive(ProcessId.ELASTICSEARCH);
     assertThat(processLauncher.processes).hasSize(1);
 
-    // leader becomes operational -> follower can start
-    appState.setOperational(ProcessId.WEB_SERVER);
-    processLauncher.waitForProcessAlive(WEB_SERVER);
-
     underTest.terminate();
   }
 
   @Test
-  public void web_server_waits_for_remote_elasticsearch_to_be_started_if_local_es_is_disabled() throws Exception {
+  public void web_follower_starts_only_when_web_leader_is_operational() throws Exception {
+    // leader takes the lock, so underTest won't get it
+    assertThat(appState.tryToLockWebLeader()).isTrue();
+    appState.setOperational(ProcessId.ELASTICSEARCH);
+
     settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
-    settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application");
     SchedulerImpl underTest = newScheduler();
     underTest.schedule();
 
-    // WEB and CE wait for ES to be up
-    assertThat(processLauncher.processes).isEmpty();
+    assertThat(processLauncher.processes).hasSize(0);
+
+    // leader becomes operational -> follower can start
+    appState.setOperational(WEB_SERVER);
 
-    // ES becomes operational on another node -> web leader can start
-    appState.setRemoteOperational(ProcessId.ELASTICSEARCH);
     processLauncher.waitForProcessAlive(WEB_SERVER);
-    assertThat(processLauncher.processes).hasSize(1);
+    processLauncher.waitForProcessAlive(COMPUTE_ENGINE);
+    assertThat(processLauncher.processes).hasSize(2);
 
     underTest.terminate();
   }
 
   @Test
-  public void compute_engine_waits_for_remote_elasticsearch_and_web_leader_to_be_started_if_local_es_is_disabled() throws Exception {
+  public void web_server_waits_for_remote_elasticsearch_to_be_started_if_local_es_is_disabled() throws Exception {
     settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
-    settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true");
-    settings.set(ProcessProperties.CLUSTER_WEB_DISABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application");
     SchedulerImpl underTest = newScheduler();
     underTest.schedule();
 
-    // CE waits for ES and WEB leader to be up
+    // WEB and CE wait for ES to be up
     assertThat(processLauncher.processes).isEmpty();
 
-    // ES and WEB leader become operational on another nodes -> CE can start
+    // ES becomes operational on another node -> web leader can start
     appState.setRemoteOperational(ProcessId.ELASTICSEARCH);
-    appState.setRemoteOperational(ProcessId.WEB_SERVER);
-
-    processLauncher.waitForProcessAlive(COMPUTE_ENGINE);
+    processLauncher.waitForProcessAlive(WEB_SERVER);
     assertThat(processLauncher.processes).hasSize(1);
 
     underTest.terminate();
@@ -303,7 +328,6 @@ public class SchedulerImplTest {
   }
 
   private Scheduler startAll() throws InterruptedException {
-    enableAllProcesses();
     SchedulerImpl scheduler = newScheduler();
     scheduler.schedule();
     processLauncher.waitForProcess(ELASTICSEARCH).operational = true;
diff --git a/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsLoopbackTest.java b/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsLoopbackTest.java
index de28db4592c..cd5534fda75 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsLoopbackTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsLoopbackTest.java
@@ -37,6 +37,7 @@ import org.sonar.process.MessageException;
 import static org.sonar.process.ProcessProperties.CLUSTER_ENABLED;
 import static org.sonar.process.ProcessProperties.CLUSTER_HOSTS;
 import static org.sonar.process.ProcessProperties.CLUSTER_NETWORK_INTERFACES;
+import static org.sonar.process.ProcessProperties.CLUSTER_NODE_TYPE;
 import static org.sonar.process.ProcessProperties.CLUSTER_SEARCH_HOSTS;
 import static org.sonar.process.ProcessProperties.JDBC_URL;
 import static org.sonar.process.ProcessProperties.SEARCH_HOST;
@@ -170,6 +171,7 @@ public class ClusterSettingsLoopbackTest {
 
     TestAppSettings testAppSettings = new TestAppSettings()
       .set(CLUSTER_ENABLED, "true")
+      .set(CLUSTER_NODE_TYPE, "search")
       .set(CLUSTER_SEARCH_HOSTS, "192.168.233.1, 192.168.233.2,192.168.233.3")
       .set(CLUSTER_HOSTS, "192.168.233.1, 192.168.233.2,192.168.233.3")
       .set(SEARCH_HOST, localAddress)
diff --git a/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsTest.java
index ed89b4685d7..ce6c3a42326 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsTest.java
@@ -33,7 +33,7 @@ import static org.sonar.process.ProcessId.ELASTICSEARCH;
 import static org.sonar.process.ProcessId.WEB_SERVER;
 import static org.sonar.process.ProcessProperties.CLUSTER_ENABLED;
 import static org.sonar.process.ProcessProperties.CLUSTER_HOSTS;
-import static org.sonar.process.ProcessProperties.CLUSTER_SEARCH_DISABLED;
+import static org.sonar.process.ProcessProperties.CLUSTER_NODE_TYPE;
 import static org.sonar.process.ProcessProperties.CLUSTER_SEARCH_HOSTS;
 import static org.sonar.process.ProcessProperties.JDBC_URL;
 import static org.sonar.process.ProcessProperties.SEARCH_HOST;
@@ -67,22 +67,53 @@ public class ClusterSettingsTest {
 
   @Test
   public void getEnabledProcesses_returns_all_processes_by_default() {
+    settings.set(CLUSTER_ENABLED, "false");
     assertThat(ClusterSettings.getEnabledProcesses(settings)).containsOnly(COMPUTE_ENGINE, ELASTICSEARCH, WEB_SERVER);
   }
 
   @Test
-  public void getEnabledProcesses_returns_all_processes_by_default_in_cluster_mode() {
+  public void getEnabledProcesses_fails_if_no_node_type_is_set_for_a_cluster_node() {
     settings.set(CLUSTER_ENABLED, "true");
+    settings.set(CLUSTER_NODE_TYPE, "");
 
-    assertThat(ClusterSettings.getEnabledProcesses(settings)).containsOnly(COMPUTE_ENGINE, ELASTICSEARCH, WEB_SERVER);
+    expectedException.expect(IllegalStateException.class);
+    expectedException.expectMessage("Invalid value for [sonar.cluster.node.type]: []");
+
+    ClusterSettings.getEnabledProcesses(settings);
   }
 
   @Test
   public void getEnabledProcesses_returns_configured_processes_in_cluster_mode() {
     settings.set(CLUSTER_ENABLED, "true");
-    settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application");
 
     assertThat(ClusterSettings.getEnabledProcesses(settings)).containsOnly(COMPUTE_ENGINE, WEB_SERVER);
+
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search");
+
+    assertThat(ClusterSettings.getEnabledProcesses(settings)).containsOnly(ELASTICSEARCH);
+  }
+
+  @Test
+  public void accept_throws_MessageException_if_no_node_type_is_configured() {
+    settings.set(CLUSTER_ENABLED, "true");
+    settings.set(CLUSTER_NODE_TYPE, "");
+
+    expectedException.expect(MessageException.class);
+    expectedException.expectMessage("Property [sonar.cluster.node.type] is mandatory");
+
+    new ClusterSettings().accept(settings.getProps());
+  }
+
+  @Test
+  public void accept_throws_MessageException_if_no_node_type_is_an_illegal_value() {
+    settings.set(CLUSTER_ENABLED, "true");
+    settings.set(CLUSTER_NODE_TYPE, "bla");
+
+    expectedException.expect(MessageException.class);
+    expectedException.expectMessage("Invalid nodeTypeValue for [sonar.cluster.node.type]: [bla], only [application, search] are allowed");
+
+    new ClusterSettings().accept(settings.getProps());
   }
 
   @Test
@@ -99,7 +130,7 @@ public class ClusterSettingsTest {
   @Test
   public void accept_throws_MessageException_if_search_enabled_with_loopback() {
     settings.set(CLUSTER_ENABLED, "true");
-    settings.set(CLUSTER_SEARCH_DISABLED, "false");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search");
     settings.set(CLUSTER_SEARCH_HOSTS, "192.168.1.1,192.168.1.2");
     settings.set(SEARCH_HOST, "::1");
 
@@ -112,7 +143,7 @@ public class ClusterSettingsTest {
   @Test
   public void accept_not_throwing_MessageException_if_search_disabled_with_loopback() {
     settings.set(CLUSTER_ENABLED, "true");
-    settings.set(CLUSTER_SEARCH_DISABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application");
     settings.set(CLUSTER_SEARCH_HOSTS, "192.168.1.1,192.168.1.2");
     settings.set(SEARCH_HOST, "127.0.0.1");
 
@@ -155,14 +186,17 @@ public class ClusterSettingsTest {
   }
 
   @Test
-  public void isLocalElasticsearchEnabled_returns_true_by_default_in_cluster_mode() {
+  public void isLocalElasticsearchEnabled_returns_true_for_a_search_node() {
+    settings.set(CLUSTER_ENABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search");
+
     assertThat(ClusterSettings.isLocalElasticsearchEnabled(settings)).isTrue();
   }
 
   @Test
-  public void isLocalElasticsearchEnabled_returns_false_if_local_es_node_is_disabled_in_cluster_mode() {
+  public void isLocalElasticsearchEnabled_returns_true_for_a_application_node() {
     settings.set(CLUSTER_ENABLED, "true");
-    settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application");
 
     assertThat(ClusterSettings.isLocalElasticsearchEnabled(settings)).isFalse();
   }
@@ -213,6 +247,7 @@ public class ClusterSettingsTest {
   private static TestAppSettings getClusterSettings() {
     TestAppSettings testAppSettings = new TestAppSettings()
       .set(CLUSTER_ENABLED, "true")
+      .set(CLUSTER_NODE_TYPE, "search")
       .set(CLUSTER_SEARCH_HOSTS, "localhost")
       .set(CLUSTER_HOSTS, "192.168.233.1, 192.168.233.2,192.168.233.3")
       .set(SEARCH_HOST, "192.168.233.1")