SONAR-4896 Check ES cluster health at startup

diff --git a/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java
index 11de7b748c4bc12215ee89119a179365429ee575..8c158ca4fa174b760a47efa0ebd83ea7b9fe1f04 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java
+++ b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java
@@ -22,7 +22,6 @@ package org.sonar.server.search;
 import com.google.common.collect.Lists;
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.ElasticSearchParseException;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -143,7 +142,7 @@ public class SearchIndex {
     IndicesAdminClient indices = client.admin().indices();
     StopWatch watch = createWatch();
     try {
-      if (! indices.exists(new IndicesExistsRequest(index)).get().isExists()) {
+      if (! indices.exists(indices.prepareExists(index).request()).get().isExists()) {
         indices.prepareCreate(index)
           .setSettings(INDEX_DEFAULT_SETTINGS)
           .addMapping("_default_", INDEX_DEFAULT_MAPPING)

diff --git a/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java b/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java
index 4b5f88c49fe64055eb578cd69481fe5daf54caa4..633249e02727632d7ee2748bcfe286cd0d3529e6 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java
+++ b/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java
@@ -20,7 +20,9 @@
 
 package org.sonar.server.search;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.io.FileUtils;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory;
@@ -47,15 +49,24 @@ public class SearchNode implements Startable {
   private static final String INSTANCE_NAME = "sonarqube";
   static final String DATA_DIR = "data/es";
 
+  private static final String DEFAULT_HEALTH_TIMEOUT = "30s";
+
   private final ServerFileSystem fileSystem;
   private final Settings settings;
+  private final String healthTimeout;
 
   // available only after startup
   private Node node;
 
   public SearchNode(ServerFileSystem fileSystem, Settings settings) {
+    this(fileSystem, settings, DEFAULT_HEALTH_TIMEOUT);
+  }
+
+  @VisibleForTesting
+  SearchNode(ServerFileSystem fileSystem, Settings settings, String healthTimeout) {
     this.fileSystem = fileSystem;
     this.settings = settings;
+    this.healthTimeout = healthTimeout;
   }
 
   @Override
@@ -74,6 +85,17 @@ public class SearchNode implements Startable {
       .settings(esSettings)
       .node();
     node.start();
+
+    if (
+      node.client().admin().cluster().prepareHealth()
+      .setWaitForYellowStatus()
+      .setTimeout(healthTimeout)
+      .execute().actionGet()
+      .getStatus() == ClusterHealthStatus.RED) {
+      throw new IllegalStateException(
+        String.format("Elasticsearch index is corrupt, please delete directory '${SonarQubeHomeDirectory}/%s' and relaunch the SonarQube server.", DATA_DIR));
+    }
+
     LOG.info("Elasticsearch started");
   }
 

diff --git a/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java b/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java
index 023109927e085c6b1de21b6c52e3c535fc96d682..340daadbaaa2caea6426787904648e4c61abb494 100644 (file)
--- a/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java
@@ -19,14 +19,20 @@
  */
 package org.sonar.server.search;
 
+import org.apache.commons.io.FileUtils;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.ClusterAdminClient;
 import org.elasticsearch.cluster.ClusterState;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.sonar.api.config.Settings;
 import org.sonar.api.platform.ServerFileSystem;
+import org.sonar.api.utils.ZipUtils;
+import org.sonar.test.TestUtils;
 
 import java.io.File;
 import java.io.IOException;
@@ -43,6 +49,7 @@ public class SearchNodeTest {
 
   ServerFileSystem fs;
   File homedir;
+  File dataDir;
 
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
@@ -52,11 +59,16 @@ public class SearchNodeTest {
     homedir = temp.newFolder();
     fs = mock(ServerFileSystem.class);
     when(fs.getHomeDir()).thenReturn(homedir);
+    dataDir = new File(homedir, SearchNode.DATA_DIR);
+  }
+
+  @After
+  public void cleanUp() {
+    FileUtils.deleteQuietly(homedir);
   }
 
   @Test
   public void start_and_stop_es_node() throws Exception {
-    File dataDir = new File(homedir, SearchNode.DATA_DIR);
     assertThat(dataDir).doesNotExist();
 
     SearchNode node = new SearchNode(fs, new Settings());
@@ -77,6 +89,32 @@ public class SearchNodeTest {
     assertThat(dataDir).exists().isDirectory();
   }
 
+  @Test
+  public void should_restore_status_on_startup() throws Exception {
+    ZipUtils.unzip(TestUtils.getResource(SearchNodeTest.class, "data-es-clean.zip"), dataDir);
+
+    SearchNode node = new SearchNode(fs, new Settings());
+    node.start();
+
+    AdminClient admin = node.client().admin();
+    assertThat(admin.indices().prepareExists("myindex").execute().actionGet().isExists()).isTrue();;
+    assertThat(admin.cluster().prepareHealth("myindex").setWaitForYellowStatus().execute().actionGet().getStatus()).isEqualTo(ClusterHealthStatus.YELLOW);
+
+    node.stop();
+  }
+
+  @Test(expected = IllegalStateException.class)
+  public void should_fail_on_corrupt_index() throws Exception {
+    ZipUtils.unzip(TestUtils.getResource(SearchNodeTest.class, "data-es-corrupt.zip"), dataDir);
+
+    SearchNode node = new SearchNode(fs, new Settings(), "5s");
+    try {
+      node.start();
+    } finally {
+      node.stop();
+    }
+  }
+
   @Test
   public void should_fail_to_get_client_if_not_started() {
     SearchNode node = new SearchNode(fs, new Settings());

diff --git a/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-clean.zip b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-clean.zip
new file mode 100644 (file)
index 0000000..b6d2860
Binary files /dev/null and b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-clean.zip differ

diff --git a/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-corrupt.zip b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-corrupt.zip
new file mode 100644 (file)
index 0000000..9f59a91
Binary files /dev/null and b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-corrupt.zip differ