From: Jean-Baptiste Lievremont Date: Mon, 6 Jan 2014 15:10:11 +0000 (+0100) Subject: SONAR-4896 Check ES cluster health at startup X-Git-Tag: 4.1~15 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=8288472e0f544f1b49a25428e638243e8a962700;p=sonarqube.git SONAR-4896 Check ES cluster health at startup --- diff --git a/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java index 11de7b748c4..8c158ca4fa1 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java @@ -22,7 +22,6 @@ package org.sonar.server.search; import com.google.common.collect.Lists; import org.apache.commons.io.IOUtils; import org.elasticsearch.ElasticSearchParseException; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -143,7 +142,7 @@ public class SearchIndex { IndicesAdminClient indices = client.admin().indices(); StopWatch watch = createWatch(); try { - if (! indices.exists(new IndicesExistsRequest(index)).get().isExists()) { + if (! indices.exists(indices.prepareExists(index).request()).get().isExists()) { indices.prepareCreate(index) .setSettings(INDEX_DEFAULT_SETTINGS) .addMapping("_default_", INDEX_DEFAULT_MAPPING) diff --git a/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java b/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java index 4b5f88c49fe..633249e0272 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java +++ b/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java @@ -20,7 +20,9 @@ package org.sonar.server.search; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.FileUtils; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.Client; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory; @@ -47,15 +49,24 @@ public class SearchNode implements Startable { private static final String INSTANCE_NAME = "sonarqube"; static final String DATA_DIR = "data/es"; + private static final String DEFAULT_HEALTH_TIMEOUT = "30s"; + private final ServerFileSystem fileSystem; private final Settings settings; + private final String healthTimeout; // available only after startup private Node node; public SearchNode(ServerFileSystem fileSystem, Settings settings) { + this(fileSystem, settings, DEFAULT_HEALTH_TIMEOUT); + } + + @VisibleForTesting + SearchNode(ServerFileSystem fileSystem, Settings settings, String healthTimeout) { this.fileSystem = fileSystem; this.settings = settings; + this.healthTimeout = healthTimeout; } @Override @@ -74,6 +85,17 @@ public class SearchNode implements Startable { .settings(esSettings) .node(); node.start(); + + if ( + node.client().admin().cluster().prepareHealth() + .setWaitForYellowStatus() + .setTimeout(healthTimeout) + .execute().actionGet() + .getStatus() == ClusterHealthStatus.RED) { + throw new IllegalStateException( + String.format("Elasticsearch index is corrupt, please delete directory '${SonarQubeHomeDirectory}/%s' and relaunch the SonarQube server.", DATA_DIR)); + } + LOG.info("Elasticsearch started"); } diff --git a/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java b/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java index 023109927e0..340daadbaaa 100644 --- a/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java +++ b/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java @@ -19,14 +19,20 @@ */ package org.sonar.server.search; +import org.apache.commons.io.FileUtils; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.cluster.ClusterState; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.api.config.Settings; import org.sonar.api.platform.ServerFileSystem; +import org.sonar.api.utils.ZipUtils; +import org.sonar.test.TestUtils; import java.io.File; import java.io.IOException; @@ -43,6 +49,7 @@ public class SearchNodeTest { ServerFileSystem fs; File homedir; + File dataDir; @Rule public TemporaryFolder temp = new TemporaryFolder(); @@ -52,11 +59,16 @@ public class SearchNodeTest { homedir = temp.newFolder(); fs = mock(ServerFileSystem.class); when(fs.getHomeDir()).thenReturn(homedir); + dataDir = new File(homedir, SearchNode.DATA_DIR); + } + + @After + public void cleanUp() { + FileUtils.deleteQuietly(homedir); } @Test public void start_and_stop_es_node() throws Exception { - File dataDir = new File(homedir, SearchNode.DATA_DIR); assertThat(dataDir).doesNotExist(); SearchNode node = new SearchNode(fs, new Settings()); @@ -77,6 +89,32 @@ public class SearchNodeTest { assertThat(dataDir).exists().isDirectory(); } + @Test + public void should_restore_status_on_startup() throws Exception { + ZipUtils.unzip(TestUtils.getResource(SearchNodeTest.class, "data-es-clean.zip"), dataDir); + + SearchNode node = new SearchNode(fs, new Settings()); + node.start(); + + AdminClient admin = node.client().admin(); + assertThat(admin.indices().prepareExists("myindex").execute().actionGet().isExists()).isTrue();; + assertThat(admin.cluster().prepareHealth("myindex").setWaitForYellowStatus().execute().actionGet().getStatus()).isEqualTo(ClusterHealthStatus.YELLOW); + + node.stop(); + } + + @Test(expected = IllegalStateException.class) + public void should_fail_on_corrupt_index() throws Exception { + ZipUtils.unzip(TestUtils.getResource(SearchNodeTest.class, "data-es-corrupt.zip"), dataDir); + + SearchNode node = new SearchNode(fs, new Settings(), "5s"); + try { + node.start(); + } finally { + node.stop(); + } + } + @Test public void should_fail_to_get_client_if_not_started() { SearchNode node = new SearchNode(fs, new Settings()); diff --git a/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-clean.zip b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-clean.zip new file mode 100644 index 00000000000..b6d286012c9 Binary files /dev/null and b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-clean.zip differ diff --git a/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-corrupt.zip b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-corrupt.zip new file mode 100644 index 00000000000..9f59a91cd37 Binary files /dev/null and b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-corrupt.zip differ