From f36785a0b40fad7acbba74d6be07059bc884c8c2 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lievremont Date: Mon, 6 Jan 2014 16:10:11 +0100 Subject: [PATCH] SONAR-4896 Check ES cluster health at startup --- .../org/sonar/server/search/SearchIndex.java | 3 +- .../org/sonar/server/search/SearchNode.java | 22 ++++++++++ .../sonar/server/search/SearchNodeTest.java | 40 +++++++++++++++++- .../search/SearchNodeTest/data-es-clean.zip | Bin 0 -> 13499 bytes .../search/SearchNodeTest/data-es-corrupt.zip | Bin 0 -> 13239 bytes 5 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-clean.zip create mode 100644 sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-corrupt.zip diff --git a/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java index 7be47572bc2..a1b6af0f0f2 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java @@ -22,7 +22,6 @@ package org.sonar.server.search; import com.google.common.collect.Lists; import org.apache.commons.io.IOUtils; import org.elasticsearch.ElasticSearchParseException; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -169,7 +168,7 @@ public class SearchIndex { IndicesAdminClient indices = client.admin().indices(); StopWatch watch = createWatch(); try { - if (! indices.exists(new IndicesExistsRequest(index)).get().isExists()) { + if (! indices.exists(indices.prepareExists(index).request()).get().isExists()) { indices.prepareCreate(index) .setSettings(INDEX_DEFAULT_SETTINGS) .addMapping("_default_", INDEX_DEFAULT_MAPPING) diff --git a/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java b/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java index 4b5f88c49fe..633249e0272 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java +++ b/sonar-server/src/main/java/org/sonar/server/search/SearchNode.java @@ -20,7 +20,9 @@ package org.sonar.server.search; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.FileUtils; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.Client; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory; @@ -47,15 +49,24 @@ public class SearchNode implements Startable { private static final String INSTANCE_NAME = "sonarqube"; static final String DATA_DIR = "data/es"; + private static final String DEFAULT_HEALTH_TIMEOUT = "30s"; + private final ServerFileSystem fileSystem; private final Settings settings; + private final String healthTimeout; // available only after startup private Node node; public SearchNode(ServerFileSystem fileSystem, Settings settings) { + this(fileSystem, settings, DEFAULT_HEALTH_TIMEOUT); + } + + @VisibleForTesting + SearchNode(ServerFileSystem fileSystem, Settings settings, String healthTimeout) { this.fileSystem = fileSystem; this.settings = settings; + this.healthTimeout = healthTimeout; } @Override @@ -74,6 +85,17 @@ public class SearchNode implements Startable { .settings(esSettings) .node(); node.start(); + + if ( + node.client().admin().cluster().prepareHealth() + .setWaitForYellowStatus() + .setTimeout(healthTimeout) + .execute().actionGet() + .getStatus() == ClusterHealthStatus.RED) { + throw new IllegalStateException( + String.format("Elasticsearch index is corrupt, please delete directory '${SonarQubeHomeDirectory}/%s' and relaunch the SonarQube server.", DATA_DIR)); + } + LOG.info("Elasticsearch started"); } diff --git a/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java b/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java index 023109927e0..340daadbaaa 100644 --- a/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java +++ b/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java @@ -19,14 +19,20 @@ */ package org.sonar.server.search; +import org.apache.commons.io.FileUtils; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.cluster.ClusterState; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.api.config.Settings; import org.sonar.api.platform.ServerFileSystem; +import org.sonar.api.utils.ZipUtils; +import org.sonar.test.TestUtils; import java.io.File; import java.io.IOException; @@ -43,6 +49,7 @@ public class SearchNodeTest { ServerFileSystem fs; File homedir; + File dataDir; @Rule public TemporaryFolder temp = new TemporaryFolder(); @@ -52,11 +59,16 @@ public class SearchNodeTest { homedir = temp.newFolder(); fs = mock(ServerFileSystem.class); when(fs.getHomeDir()).thenReturn(homedir); + dataDir = new File(homedir, SearchNode.DATA_DIR); + } + + @After + public void cleanUp() { + FileUtils.deleteQuietly(homedir); } @Test public void start_and_stop_es_node() throws Exception { - File dataDir = new File(homedir, SearchNode.DATA_DIR); assertThat(dataDir).doesNotExist(); SearchNode node = new SearchNode(fs, new Settings()); @@ -77,6 +89,32 @@ public class SearchNodeTest { assertThat(dataDir).exists().isDirectory(); } + @Test + public void should_restore_status_on_startup() throws Exception { + ZipUtils.unzip(TestUtils.getResource(SearchNodeTest.class, "data-es-clean.zip"), dataDir); + + SearchNode node = new SearchNode(fs, new Settings()); + node.start(); + + AdminClient admin = node.client().admin(); + assertThat(admin.indices().prepareExists("myindex").execute().actionGet().isExists()).isTrue();; + assertThat(admin.cluster().prepareHealth("myindex").setWaitForYellowStatus().execute().actionGet().getStatus()).isEqualTo(ClusterHealthStatus.YELLOW); + + node.stop(); + } + + @Test(expected = IllegalStateException.class) + public void should_fail_on_corrupt_index() throws Exception { + ZipUtils.unzip(TestUtils.getResource(SearchNodeTest.class, "data-es-corrupt.zip"), dataDir); + + SearchNode node = new SearchNode(fs, new Settings(), "5s"); + try { + node.start(); + } finally { + node.stop(); + } + } + @Test public void should_fail_to_get_client_if_not_started() { SearchNode node = new SearchNode(fs, new Settings()); diff --git a/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-clean.zip b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-clean.zip new file mode 100644 index 0000000000000000000000000000000000000000..b6d286012c948be21694bff90f6407db7b0ef037 GIT binary patch literal 13499 zcmd6tdrVVT9LH}dMFl5JKzs~cL_{2qz9ulUFz1U(G*})QO^{M9j0#0)*$|TropTCq zF>JcXjG9H9LWqlxOp|47i)Ic{gUjZYnQ7+YAIqk6n#GA5JC}NT?rI ze&65o({s;{f_!y&B*VQDo8~SV`OA~vG|V$hg~e&fEH0YBsHc6}XMTLH&)nD)#i+vn zQZY<;9Quz1qTe2kzui%1w=~o@ma~+G;zt{@*Hy3%N^=Qf=CqUsXNnu#;BZ=;kho`8 z&cB#~(#pN(`n6kCQ&(=ON!NJ_^b;zQSAkwgjnq8aS<5;t>0B`%?Rb@Ka8%dXzuM_! zYd6&(_Z*LghK9uJI7{4(YI{YsmAViq{tee|MiOijWnog~3((eH3z3l74cu6bADKo~ ze^>0BC*6=mp)}Hc>3XU5=)PkvD43t0KL1Mp)k50cGTUE>2osNtu)-I(yG(0?<8!^6Y`@jfNdHq{GThvCRy&S!B z==C>9LgG~K!K5pz4|E*-uw#w6zlGU-vF2|3N9RBL<-p?y5BuXv7h9w9)r$>LN!Hk; z{5_{{X3VIQAio~srnA9fchuBXQO%hdSV zwafRNmm+s(HB?;cNjsjF7vEJ<(6_3m=%g9lLG-7tEH)owTU5)QZCKTNrpJ|w0#M|+ z%Q*I|G6sxQ9;HxXl`&AP!a}}TwID#ZJJ^f=&8n9-o*1CG-SO5}j8%rDJr7E!ymeb* z?f|ztM)|Dry@CL+N}%P7RSWoAP|)p8hWz^_@WqR%H*4V)MC7~Z{XAGl#`vGQ_G|e+ z-|bt!fBy&V>s;MEX=jdQ8*=aE<=30aOTl!GaX;* z`n%e7VcM>xsTU?4{dOR^)g0BJe=oKzapK1>9XWjGx8#J*u*)m!<7ZQ>TC5p0{sb z<yXJS68OZV}AVi_Py_qX_rO37-1c-b>8S} z@7q?C+->XJHv905wiWuqw#26WCkvlT{9$^<*}bj#tu4*(+qT}ST3^1Zvb^5rI&ri2 z4K->>RoG0m??%xuQ&AGRR}AyVBJR&8n=6fbb2aKVPux^AVJa7!5^k~JM&k*iJ4YME z8!fo$1j6)nG1J_J18#gKVSFCi`igA z)ElTp-lb6W&=Q{EOd{l@a6riTGFnQMkC z03LMr&k#y=9bbR(r>FDu4k$sIjj@3E@u2$|Ha=`}(ItFnyiTQy9Z;GyhmgxMPgvaY zN~-2^&NN~106E3ZG&K~eijhV&1rV}Qq_M$8eUpY;WJwjB@AzubdzwMGGc=?jS6IT4 z+mrw<0d_zM(j*{XWX~8LbgE$^&opmR>0$?jq~VrUQeEJC^fNrSM%*q$Gfj3DfP+u2 cuY`rK1#9{pdkwn$Vwf!SYX}+dn$G>}KZgFDW&i*H literal 0 HcmV?d00001 diff --git a/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-corrupt.zip b/sonar-server/src/test/resources/org/sonar/server/search/SearchNodeTest/data-es-corrupt.zip new file mode 100644 index 0000000000000000000000000000000000000000..9f59a91cd37c4c0524e020ebfffe263aa48de4b6 GIT binary patch literal 13239 zcmd6tdrVVT9LH~g@^s(?MBG5bk0VR?d>*GXGfQrCJdD99mv++MB53&rHF-75e7^b54g|nFgYM~ zXEf)1la1QSyt4epZE0;cn_3Ih>;nBW<+VW1=Ef=?4Ytt^QvtJ>kB%ImJMETs+n0k5 zy6v@Al%D<3!oq_1I&KnA!(wZ)G!idFw*SCwJxGG?CR~_Y`2wVCuYyR(+zrfFMK4XG z;`)WeE$-b=OrW&1`wBEt+hhE$p{6FUsvz&?_1Rj(hX&QWA|@$`zMHkazCU8;(9YT^ zt3rHl6he}cx0GDBYvkjhwoo;EPZ4Ud^7TPAIys)^qSQZ&=9GK%F)p$@yKiv$ouT^K52uu7|m80x|h z5d#aYEHQ8v%OXZIIWau--NYzEwD57P5T79cG1P^4cnOCIKn$K177Z~mG=xP0h@lA? zF<7P5L<~)+h=GMxmKZpTWf5aPIWau--NYylA~7@}BnD4wg@~aIJ~6Z*BL=Iqnuwtd z6)~{T$`S)-u`FV=k`u#I-%X4cgGdamd}449eYanZ)m5MezL(GoaK_+i!HLmmG~TD1 zlE+mE#os#b$HqJLU9ZpIc;L7ed%U=_>Dom8`TWY{3k@|h)f07>4CtBtdhV`z!x-JC zsMy?DJv}+$EJFo|pUVKxHoD+*M;9{guu7|mJGxMD2MhV;PEm+%LvR+$;*O1+JD&P( z?$iX4J39H?@qPUc;0{mA7k7#zZve%Ku~q(k4EUl#KV7`-CZg+~==BU(N5&{$p8wVS z@2%lIr%%6g@)hU!ME>MhiMH%sWmSjXtiND?W4m?l!49il-@LcidGWk)M{m!_l>Yn< z=ie6R)zmk4--Z<#40VK;VlOzKps%OCwdP&qu@ zJUTGCF{-Dxq=8oMzVdZSeaWda4RuG3jx5?{cjf0$KmGU1z3<0V6_L+HHqKc`ZqJ;Y z>95Niw~q8@oPFZhPEGBxwC>ZFYM)N~abwe!Q$tlleZ6m65C7b<$6VcP?yx#9-kClW zjOAq(GQIymM)I{cME%nEC=3lpKqjb48UXG1;s02Z%G8~HU;J||>3xW=M2gPm< zJamd6^bca8ncIDM@N_}&9OQT{^_6ReD{mF|P9{%7=<(pSqo{L-`}}Hfj*mPo)DiY; zlc1icFwuHATL&(ZEf6vK7vU@(xLB@0>Af#pk#b`D<#TREHr!gLvb7~>ie+6SU)n85qLiQWY!@=N`wM< zYPdyrbwGILkxIaYcW={&pTJ4-1eik)l4mR+z*~YwumhBlfco)SgowVCtS+Q?qChJmbM%2n4559+Uw8{x)9=_bmu?hQ OjQ%X7Qq(AVQPls8r2sDg literal 0 HcmV?d00001 -- 2.39.5