SONAR-5687 - Slave fails to start if master does not have replication > 0

2 files changed, 130 insertions, 11 deletions

diff --git a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java
index 86e2c19b90e..9a32277c96f 100644
--- a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java
+++ b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java
@@ -21,7 +21,9 @@ package org.sonar.search;
 
 import org.apache.commons.lang.StringUtils;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.hppc.cursors.ObjectCursor;
 import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.node.internal.InternalNode;
 import org.slf4j.LoggerFactory;
@@ -40,13 +42,15 @@ import java.util.Set;
 
 public class SearchServer implements Monitored {
 
+  public static final String WRONG_MASTER_REPLICATION_FACTOR = "Index configuration is not set to cluster. Please start the master node with 'sonar.cluster.activation=true'";
+
   public static final String CLUSTER_ACTIVATION = "sonar.cluster.activation";
   public static final String SONAR_NODE_NAME = "sonar.node.name";
   public static final String ES_PORT_PROPERTY = "sonar.search.port";
   public static final String ES_CLUSTER_PROPERTY = "sonar.cluster.name";
   public static final String ES_CLUSTER_INET = "sonar.cluster.master";
-  public static final String ES_MARVEL_HOST = "sonar.search.marvel";
 
+  public static final String ES_MARVEL_HOST = "sonar.search.marvel";
   public static final String SONAR_PATH_HOME = "sonar.path.home";
   public static final String SONAR_PATH_DATA = "sonar.path.data";
   public static final String SONAR_PATH_TEMP = "sonar.path.temp";
@@ -113,10 +117,6 @@ public class SearchServer implements Monitored {
       .put("path.logs", esLogDir().getAbsolutePath());
 
     if (!nodes.isEmpty()) {
-      // When sonar has a master, for the cluster mode
-      // see https://jira.codehaus.org/browse/SONAR-5687
-      replicationFactor = 1;
-
       LoggerFactory.getLogger(SearchServer.class).info("Joining ES cluster with master: {}", nodes);
       esSettings.put("discovery.zen.ping.unicast.hosts", StringUtils.join(nodes, ","));
       esSettings.put("node.master", false);
@@ -159,6 +159,20 @@ public class SearchServer implements Monitored {
     node = new InternalNode(esSettings.build(), true);
     node.start();
 
+    // When joining a cluster, make sur the master(s) have a
+    // replication factor on all indices > 0
+    if (!nodes.isEmpty()) {
+      for (ObjectCursor<Settings> settingCursor : node.client().admin().indices()
+        .prepareGetSettings().get().getIndexToSettings().values()) {
+        Settings settings = settingCursor.value;
+        String clusterReplicationFactor = settings.get("index.number_of_replicas", "-1");
+        if (Integer.parseInt(clusterReplicationFactor) <= 0) {
+          node.stop();
+          throw new IllegalStateException(WRONG_MASTER_REPLICATION_FACTOR);
+        }
+      }
+    }
+
     node.client().admin().indices()
       .preparePutTemplate("default")
       .setTemplate("*")
diff --git a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java
index 208a66a504d..a6677dc9ff5 100644
--- a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java
+++ b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java
@@ -26,6 +26,7 @@ import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -41,6 +42,8 @@ public class SearchServerTest {
 
   Integer port;
   String cluster;
+  SearchServer searchServer;
+  Client client;
 
   @Before
   public void setUp() throws Exception {
@@ -48,9 +51,23 @@ public class SearchServerTest {
     cluster = "unitTest";
   }
 
+  @After
+  public void tearDown() throws Exception {
+    if (searchServer != null) {
+      searchServer.stop();
+      searchServer.awaitStop();
+    }
+    if (client != null) {
+      client.close();
+    }
+  }
+
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
 
+  @Rule
+  public TemporaryFolder temp2 = new TemporaryFolder();
+
   @Test(timeout = 15000L)
   public void start_stop_server() throws Exception {
 
@@ -59,16 +76,17 @@ public class SearchServerTest {
     props.put(SearchServer.ES_CLUSTER_PROPERTY, cluster);
     props.put(SearchServer.SONAR_PATH_HOME, temp.getRoot().getAbsolutePath());
 
-    SearchServer searchServer = new SearchServer(new Props(props));
+    searchServer = new SearchServer(new Props(props));
     assertThat(searchServer).isNotNull();
 
     searchServer.start();
     assertThat(searchServer.isReady()).isTrue();
 
-    Client client = getSearchClient();
+    client = getSearchClient();
 
     searchServer.stop();
     searchServer.awaitStop();
+    searchServer = null;
     try {
       assertThat(client.admin().cluster().prepareClusterStats().get().getStatus()).isNotEqualTo(ClusterHealthStatus.GREEN);
     } catch (NoNodeAvailableException exception) {
@@ -84,19 +102,22 @@ public class SearchServerTest {
     props.put(SearchServer.ES_CLUSTER_PROPERTY, cluster);
     props.put(SearchServer.SONAR_PATH_HOME, temp.getRoot().getAbsolutePath());
 
-    SearchServer searchServer = new SearchServer(new Props(props));
+    searchServer = new SearchServer(new Props(props));
     assertThat(searchServer).isNotNull();
 
     searchServer.start();
     assertThat(searchServer.isReady()).isTrue();
 
-    Client client = getSearchClient();
+    client = getSearchClient();
     client.admin().indices().prepareCreate("test").get();
 
     assertThat(client.admin().indices().prepareGetSettings("test")
       .get()
       .getSetting("test", "index.number_of_replicas"))
       .isEqualTo("0");
+
+    searchServer.stop();
+    searchServer.awaitStop();
   }
 
   @Test
@@ -108,19 +129,103 @@ public class SearchServerTest {
     props.put(SearchServer.ES_CLUSTER_PROPERTY, cluster);
     props.put(SearchServer.SONAR_PATH_HOME, temp.getRoot().getAbsolutePath());
 
-    SearchServer searchServer = new SearchServer(new Props(props));
+    searchServer = new SearchServer(new Props(props));
     assertThat(searchServer).isNotNull();
 
     searchServer.start();
     assertThat(searchServer.isReady()).isTrue();
 
-    Client client = getSearchClient();
+    client = getSearchClient();
     client.admin().indices().prepareCreate("test").get();
 
     assertThat(client.admin().indices().prepareGetSettings("test")
       .get()
       .getSetting("test", "index.number_of_replicas"))
       .isEqualTo("1");
+
+    searchServer.stop();
+    searchServer.awaitStop();
+  }
+
+  @Test
+  public void slave_success_replication() throws Exception {
+
+    Properties props = new Properties();
+    props.put(SearchServer.CLUSTER_ACTIVATION, Boolean.TRUE.toString());
+    props.put(SearchServer.ES_PORT_PROPERTY, port.toString());
+    props.put(SearchServer.SONAR_NODE_NAME, "MASTER");
+    props.put(SearchServer.ES_CLUSTER_PROPERTY, cluster);
+    props.put(SearchServer.SONAR_PATH_HOME, temp.getRoot().getAbsolutePath());
+    searchServer = new SearchServer(new Props(props));
+    assertThat(searchServer).isNotNull();
+
+    searchServer.start();
+    assertThat(searchServer.isReady()).isTrue();
+
+    client = getSearchClient();
+    client.admin().indices().prepareCreate("test").get();
+
+    // start a slave
+    props = new Properties();
+    props.put(SearchServer.ES_CLUSTER_INET, "localhost:" + port);
+    props.put(SearchServer.SONAR_NODE_NAME, "SLAVE");
+    props.put(SearchServer.ES_PORT_PROPERTY, NetworkUtils.freePort() + "");
+    props.put(SearchServer.ES_CLUSTER_PROPERTY, cluster);
+    props.put(SearchServer.SONAR_PATH_HOME, temp2.getRoot().getAbsolutePath());
+    SearchServer slaveServer = new SearchServer(new Props(props));
+    assertThat(slaveServer).isNotNull();
+
+    slaveServer.start();
+    assertThat(slaveServer.isReady()).isTrue();
+
+    assertThat(client.admin().cluster().prepareClusterStats().get()
+      .getNodesStats().getCounts().getTotal()).isEqualTo(2);
+
+    searchServer.stop();
+    slaveServer.stop();
+    searchServer.awaitStop();
+    slaveServer.awaitStop();
+  }
+
+  @Test
+  public void slave_failed_replication() throws Exception {
+
+    Properties props = new Properties();
+    props.put(SearchServer.CLUSTER_ACTIVATION, Boolean.FALSE.toString());
+    props.put(SearchServer.ES_PORT_PROPERTY, port.toString());
+    props.put(SearchServer.SONAR_NODE_NAME, "MASTER");
+    props.put(SearchServer.ES_CLUSTER_PROPERTY, cluster);
+    props.put(SearchServer.SONAR_PATH_HOME, temp.getRoot().getAbsolutePath());
+    searchServer = new SearchServer(new Props(props));
+    assertThat(searchServer).isNotNull();
+
+    searchServer.start();
+    assertThat(searchServer.isReady()).isTrue();
+
+    client = getSearchClient();
+    client.admin().indices().prepareCreate("test").get();
+
+    // start a slave
+    props = new Properties();
+    props.put(SearchServer.ES_CLUSTER_INET, "localhost:" + port);
+    props.put(SearchServer.SONAR_NODE_NAME, "SLAVE");
+    props.put(SearchServer.ES_PORT_PROPERTY, NetworkUtils.freePort() + "");
+    props.put(SearchServer.ES_CLUSTER_PROPERTY, cluster);
+    props.put(SearchServer.SONAR_PATH_HOME, temp2.getRoot().getAbsolutePath());
+    SearchServer slaveServer = new SearchServer(new Props(props));
+    assertThat(slaveServer).isNotNull();
+
+    try {
+      slaveServer.start();
+    } catch (Exception e) {
+      assertThat(e.getMessage()).isEqualTo(SearchServer.WRONG_MASTER_REPLICATION_FACTOR);
+    }
+
+    assertThat(client.admin().cluster().prepareClusterStats().get()
+      .getNodesStats().getCounts().getTotal()).isEqualTo(1);
+
+    slaveServer.stop();
+    slaveServer.awaitStop();
   }
 
   private Client getSearchClient() {