SONAR-9134 Define the minimum number of ES master nodes

In cluster mode, there'll be 2 master nodes by default.
In standalone mode, only 1 master node by default
A new property sonar.search.initialStateTimeout allows to handle the startup
timeout.

6 files changed, 152 insertions, 41 deletions

diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java
index 0f2258f5cf8..959788fb933 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java
@@ -67,6 +67,8 @@ public class ProcessProperties {
   public static final String SEARCH_JAVA_OPTS = "sonar.search.javaOpts";
   public static final String SEARCH_JAVA_ADDITIONAL_OPTS = "sonar.search.javaAdditionalOpts";
   public static final String SEARCH_REPLICAS = "sonar.search.replicas";
+  public static final String SEARCH_MINIMUM_MASTER_NODES = "sonar.search.minimumMasterNodes";
+  public static final String SEARCH_INITIAL_STATE_TIMEOUT = "sonar.search.initialStateTimeout";
 
   public static final String WEB_JAVA_OPTS = "sonar.web.javaOpts";
 
diff --git a/server/sonar-search/pom.xml b/server/sonar-search/pom.xml
index bf21503aa2d..e3e5a3ed316 100644
--- a/server/sonar-search/pom.xml
+++ b/server/sonar-search/pom.xml
@@ -63,6 +63,11 @@
       <artifactId>assertj-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
 </project>
diff --git a/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java b/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java
index aae8545f3b1..bb882f38a0d 100644
--- a/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java
+++ b/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java
@@ -117,10 +117,10 @@ public class EsSettings implements EsSettingsMBean {
     LOGGER.info("Elasticsearch listening on {}:{}", host, port);
 
     // disable multicast
-    builder.put("discovery.zen.ping.multicast.enabled", "false");
-    builder.put("transport.tcp.port", port);
-    builder.put("transport.host", host.getHostAddress());
-    builder.put("network.host", host.getHostAddress());
+    builder.put("discovery.zen.ping.multicast.enabled", "false")
+      .put("transport.tcp.port", port)
+      .put("transport.host", host.getHostAddress())
+      .put("network.host", host.getHostAddress());
 
     // Elasticsearch sets the default value of TCP reuse address to true only on non-MSWindows machines, but why ?
     builder.put("network.tcp.reuse_address", true);
@@ -132,11 +132,11 @@ public class EsSettings implements EsSettingsMBean {
     } else {
       LOGGER.warn("Elasticsearch HTTP connector is enabled on port {}. MUST NOT BE USED FOR PRODUCTION", httpPort);
       // see https://github.com/lmenezes/elasticsearch-kopf/issues/195
-      builder.put("http.cors.enabled", true);
-      builder.put("http.cors.allow-origin", "*");
-      builder.put("http.enabled", true);
-      builder.put("http.host", host.getHostAddress());
-      builder.put("http.port", httpPort);
+      builder.put("http.cors.enabled", true)
+        .put("http.cors.allow-origin", "*")
+        .put("http.enabled", true)
+        .put("http.host", host.getHostAddress())
+        .put("http.port", httpPort);
     }
   }
 
@@ -158,20 +158,23 @@ public class EsSettings implements EsSettingsMBean {
   }
 
   private void configureCluster(Settings.Builder builder) {
-    int replicationFactor = props.valueAsInt(ProcessProperties.SEARCH_REPLICAS, 0);
+    // Default value in a standalone mode, not overridable
+    int replicationFactor = 0;
+    int minimumMasterNodes = 1;
+    String initialStateTimeOut = "30s";
 
     if (clusterEnabled) {
-      if (!props.contains(ProcessProperties.SEARCH_REPLICAS)) {
-        // In data center edition there is 3 nodes, so if not defined, replicationFactor default value is 1
-        replicationFactor = 1;
-      }
+      replicationFactor = props.valueAsInt(ProcessProperties.SEARCH_REPLICAS, 1);
+      minimumMasterNodes = props.valueAsInt(ProcessProperties.SEARCH_MINIMUM_MASTER_NODES, 2);
+      initialStateTimeOut = props.value(ProcessProperties.SEARCH_INITIAL_STATE_TIMEOUT, "120s");
 
       String hosts = props.value(ProcessProperties.CLUSTER_SEARCH_HOSTS, "");
       LOGGER.info("Elasticsearch cluster enabled. Connect to hosts [{}]", hosts);
       builder.put("discovery.zen.ping.unicast.hosts", hosts);
     }
 
-    builder.put("discovery.zen.minimum_master_nodes", 1)
+    builder.put("discovery.zen.minimum_master_nodes", minimumMasterNodes)
+      .put("discovery.initial_state_timeout", initialStateTimeOut)
       .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicationFactor)
       .put("cluster.name", getClusterName())
       .put("cluster.routing.allocation.awareness.attributes", "rack_id")
diff --git a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java
index 6929b44212a..21e75193376 100644
--- a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java
+++ b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java
@@ -21,9 +21,12 @@ package org.sonar.search;
 
 import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.sonar.process.Jmx;
 import org.sonar.process.MinimumViableSystem;
 import org.sonar.process.Monitored;
@@ -31,7 +34,11 @@ import org.sonar.process.ProcessEntryPoint;
 import org.sonar.process.Props;
 
 public class SearchServer implements Monitored {
+  // VisibleForTesting
+  protected static Logger LOGGER = LoggerFactory.getLogger(SearchServer.class);
 
+  private static final String MIMINUM_MASTER_NODES = "discovery.zen.minimum_master_nodes";
+  private static final String INITIAL_STATE_TIMEOUT = "discovery.initial_state_timeout";
   private final EsSettings settings;
   private Node node;
 
@@ -45,7 +52,14 @@ public class SearchServer implements Monitored {
   public void start() {
     Jmx.register(EsSettingsMBean.OBJECT_NAME, settings);
     initBootstrap();
-    node = NodeBuilder.nodeBuilder().settings(settings.build()).build();
+    Settings esSettings = settings.build();
+    if (esSettings.getAsInt(MIMINUM_MASTER_NODES, 1) >= 2) {
+      LOGGER.info("Elasticsearch is waiting {} for {} node(s) to be up to start.",
+        esSettings.get(INITIAL_STATE_TIMEOUT),
+        esSettings.get(MIMINUM_MASTER_NODES)
+      );
+    }
+    node = NodeBuilder.nodeBuilder().settings(esSettings).build();
     node.start();
   }
 
diff --git a/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java b/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java
index b03f58e55b5..9abbb3248b1 100644
--- a/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java
+++ b/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java
@@ -68,6 +68,8 @@ public class EsSettingsTest {
 
     assertThat(generated.get("index.number_of_replicas")).isEqualTo("0");
     assertThat(generated.get("discovery.zen.ping.unicast.hosts")).isNull();
+    assertThat(generated.get("discovery.zen.minimum_master_nodes")).isEqualTo("1");
+    assertThat(generated.get("discovery.initial_state_timeout")).isEqualTo("30s");
   }
 
   @Test
@@ -75,7 +77,7 @@ public class EsSettingsTest {
     File dataDir = temp.newFolder();
     File logDir = temp.newFolder();
     File tempDir = temp.newFolder();
-    Props props = minProps();
+    Props props = minProps(false);
     props.set(ProcessProperties.PATH_DATA, dataDir.getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, logDir.getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, tempDir.getAbsolutePath());
@@ -89,28 +91,75 @@ public class EsSettingsTest {
 
   @Test
   public void cluster_is_enabled() throws Exception {
-    Props props = minProps();
-    props.set(ProcessProperties.CLUSTER_ENABLED, "true");
+    Props props = minProps(true);
     props.set(ProcessProperties.CLUSTER_SEARCH_HOSTS, "1.2.3.4:9000,1.2.3.5:8080");
     Settings settings = new EsSettings(props).build();
 
     assertThat(settings.get("index.number_of_replicas")).isEqualTo("1");
     assertThat(settings.get("discovery.zen.ping.unicast.hosts")).isEqualTo("1.2.3.4:9000,1.2.3.5:8080");
+    assertThat(settings.get("discovery.zen.minimum_master_nodes")).isEqualTo("2");
+    assertThat(settings.get("discovery.initial_state_timeout")).isEqualTo("120s");
   }
 
   @Test
-  public void cluster_is_enabled_with_defined_replicas() throws Exception {
-    Props props = minProps();
-    props.set(ProcessProperties.CLUSTER_ENABLED, "true");
+  public void incorrect_values_of_minimum_master_nodes() throws Exception {
+    Props props = minProps(true);
+    props.set(ProcessProperties.SEARCH_MINIMUM_MASTER_NODES, "ꝱꝲꝳପ");
+
+    expectedException.expect(IllegalStateException.class);
+    expectedException.expectMessage("Value of property sonar.search.minimumMasterNodes is not an integer:");
+    Settings settings = new EsSettings(props).build();
+  }
+
+  @Test
+  public void cluster_is_enabled_with_defined_minimum_master_nodes() throws Exception {
+    Props props = minProps(true);
+    props.set(ProcessProperties.SEARCH_MINIMUM_MASTER_NODES, "5");
+    Settings settings = new EsSettings(props).build();
+
+    assertThat(settings.get("discovery.zen.minimum_master_nodes")).isEqualTo("5");
+  }
+
+  @Test
+  public void cluster_is_enabled_with_defined_initialTimeout() throws Exception {
+    Props props = minProps(true);
+    props.set(ProcessProperties.SEARCH_INITIAL_STATE_TIMEOUT, "10s");
+    Settings settings = new EsSettings(props).build();
+
+    assertThat(settings.get("discovery.initial_state_timeout")).isEqualTo("10s");
+  }
+
+  @Test
+  public void in_standalone_initialTimeout_is_not_overridable() throws Exception {
+    Props props = minProps(false);
+    props.set(ProcessProperties.SEARCH_INITIAL_STATE_TIMEOUT, "10s");
+    Settings settings = new EsSettings(props).build();
+
+    assertThat(settings.get("discovery.initial_state_timeout")).isEqualTo("30s");
+  }
+
+  @Test
+  public void in_standalone_minimumMasterNodes_is_not_overridable() throws Exception {
+    Props props = minProps(false);
+    props.set(ProcessProperties.SEARCH_MINIMUM_MASTER_NODES, "5");
+    Settings settings = new EsSettings(props).build();
+
+    assertThat(settings.get("discovery.zen.minimum_master_nodes")).isEqualTo("1");
+  }
+
+
+  @Test
+  public void in_standalone_searchReplicas_is_not_overridable() throws Exception {
+    Props props = minProps(false);
     props.set(ProcessProperties.SEARCH_REPLICAS, "5");
     Settings settings = new EsSettings(props).build();
 
-    assertThat(settings.get("index.number_of_replicas")).isEqualTo("5");
+    assertThat(settings.get("index.number_of_replicas")).isEqualTo("0");
   }
 
   @Test
-  public void cluster_not_enabled_but_replicas_are_defined() throws Exception {
-    Props props = minProps();
+  public void cluster_is_enabled_with_defined_replicas() throws Exception {
+    Props props = minProps(true);
     props.set(ProcessProperties.SEARCH_REPLICAS, "5");
     Settings settings = new EsSettings(props).build();
 
@@ -119,7 +168,8 @@ public class EsSettingsTest {
 
   @Test
   public void incorrect_values_of_replicas() throws Exception {
-    Props props = minProps();
+    Props props = minProps(true);
+
     props.set(ProcessProperties.SEARCH_REPLICAS, "ꝱꝲꝳପ");
 
     expectedException.expect(IllegalStateException.class);
@@ -129,7 +179,7 @@ public class EsSettingsTest {
 
   @Test
   public void enable_marvel() throws Exception {
-    Props props = minProps();
+    Props props = minProps(false);
     props.set(EsSettings.PROP_MARVEL_HOSTS, "127.0.0.2,127.0.0.3");
     Settings settings = new EsSettings(props).build();
 
@@ -138,7 +188,7 @@ public class EsSettingsTest {
 
   @Test
   public void enable_http_connector() throws Exception {
-    Props props = minProps();
+    Props props = minProps(false);
     props.set(ProcessProperties.SEARCH_HTTP_PORT, "9010");
     Settings settings = new EsSettings(props).build();
 
@@ -149,7 +199,7 @@ public class EsSettingsTest {
   
   @Test
   public void enable_http_connector_different_host() throws Exception {
-    Props props = minProps();
+    Props props = minProps(false);
     props.set(ProcessProperties.SEARCH_HTTP_PORT, "9010");
     props.set(ProcessProperties.SEARCH_HOST, "127.0.0.2");
     Settings settings = new EsSettings(props).build();
@@ -159,11 +209,12 @@ public class EsSettingsTest {
     assertThat(settings.get("http.enabled")).isEqualTo("true");
   }
 
-  private Props minProps() throws IOException {
+  private Props minProps(boolean cluster) throws IOException {
     File homeDir = temp.newFolder();
     Props props = new Props(new Properties());
     ProcessProperties.completeDefaults(props);
     props.set(ProcessProperties.PATH_HOME, homeDir.getAbsolutePath());
+    props.set(ProcessProperties.CLUSTER_ENABLED, Boolean.toString(cluster));
     return props;
   }
 }
diff --git a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java
index 5b5fc075f4c..88c1c1960a1 100644
--- a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java
+++ b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.search;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.util.Properties;
 import org.elasticsearch.client.Client;
@@ -34,6 +35,7 @@ import org.junit.rules.DisableOnDebug;
 import org.junit.rules.TemporaryFolder;
 import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
+import org.slf4j.Logger;
 import org.sonar.process.Monitored;
 import org.sonar.process.NetworkUtils;
 import org.sonar.process.ProcessEntryPoint;
@@ -42,6 +44,10 @@ import org.sonar.process.Props;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 
 public class SearchServerTest {
 
@@ -70,24 +76,41 @@ public class SearchServerTest {
   }
 
   @Test
-  public void start_stop_server() throws Exception {
-    Props props = new Props(new Properties());
-    // the following properties have always default values (see ProcessProperties)
-    InetAddress host = InetAddress.getLoopbackAddress();
-    props.set(ProcessProperties.SEARCH_HOST, host.getHostAddress());
-    props.set(ProcessProperties.SEARCH_PORT, String.valueOf(port));
-    props.set(ProcessProperties.CLUSTER_NAME, A_CLUSTER_NAME);
-    props.set(EsSettings.CLUSTER_SEARCH_NODE_NAME, A_NODE_NAME);
-    props.set(ProcessProperties.PATH_HOME, temp.newFolder().getAbsolutePath());
-    props.set(ProcessEntryPoint.PROPERTY_SHARED_PATH, temp.newFolder().getAbsolutePath());
+  public void log_information_on_startup() throws IOException {
+    Props props = getClusterProperties();
+    props.set(ProcessProperties.CLUSTER_ENABLED, "true");
+    props.set(ProcessProperties.SEARCH_MINIMUM_MASTER_NODES, "2");
+    // Set the timeout to 1sec
+    props.set(ProcessProperties.SEARCH_INITIAL_STATE_TIMEOUT, "1s");
+    underTest = new SearchServer(props);
+    Logger logger = mock(Logger.class);
+    underTest.LOGGER = logger;
+    underTest.start();
+    verify(logger).info(eq("Elasticsearch is waiting {} for {} node(s) to be up to start."), eq("1s"), eq("2"));
+  }
 
+  @Test
+  public void no_log_information_on_startup() throws IOException {
+    Props props = getClusterProperties();
+    props.set(ProcessProperties.SEARCH_MINIMUM_MASTER_NODES, "1");
+    // Set the timeout to 1sec
+    props.set(ProcessProperties.SEARCH_INITIAL_STATE_TIMEOUT, "1s");
     underTest = new SearchServer(props);
+    Logger logger = mock(Logger.class);
+    underTest.LOGGER = logger;
+    underTest.start();
+    verify(logger, never()).info(eq("Elasticsearch is waiting {} for {} node(s) to be up to start."), eq("1s"), eq("2"));
+  }
+
+  @Test
+  public void start_stop_server() throws Exception {
+    underTest = new SearchServer(getClusterProperties());
     underTest.start();
     assertThat(underTest.getStatus()).isEqualTo(Monitored.Status.OPERATIONAL);
 
     Settings settings = Settings.builder().put("cluster.name", A_CLUSTER_NAME).build();
     client = TransportClient.builder().settings(settings).build()
-      .addTransportAddress(new InetSocketTransportAddress(host, port));
+      .addTransportAddress(new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), port));
     assertThat(client.admin().cluster().prepareClusterStats().get().getStatus()).isEqualTo(ClusterHealthStatus.GREEN);
 
     underTest.stop();
@@ -100,4 +123,17 @@ public class SearchServerTest {
       // ok
     }
   }
+
+  private Props getClusterProperties() throws IOException {
+    Props props = new Props(new Properties());
+    // the following properties have always default values (see ProcessProperties)
+    InetAddress host = InetAddress.getLoopbackAddress();
+    props.set(ProcessProperties.SEARCH_HOST, host.getHostAddress());
+    props.set(ProcessProperties.SEARCH_PORT, String.valueOf(port));
+    props.set(ProcessProperties.CLUSTER_NAME, A_CLUSTER_NAME);
+    props.set(EsSettings.CLUSTER_SEARCH_NODE_NAME, A_NODE_NAME);
+    props.set(ProcessProperties.PATH_HOME, temp.newFolder().getAbsolutePath());
+    props.set(ProcessEntryPoint.PROPERTY_SHARED_PATH, temp.newFolder().getAbsolutePath());
+    return props;
+  }
 }