From: Jacek Date: Mon, 27 Sep 2021 13:20:23 +0000 (+0200) Subject: SONAR-15418 Suppress Elasticsearch timeout exception when waiting for it to be UP X-Git-Tag: 8.9.5.50698~2 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=6e40b884141ea72cc75a57d7b910929685cff126;p=sonarqube.git SONAR-15418 Suppress Elasticsearch timeout exception when waiting for it to be UP (cherry picked from commit 4cc250a69e9f592cb6977aa4ce15ed50a83856be) --- diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java index bf5bad5d183..356bbbe2b1d 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java +++ b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java @@ -19,7 +19,10 @@ */ package org.sonar.application.process; +import com.google.common.collect.ImmutableSet; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.rest.RestStatus; @@ -37,15 +40,21 @@ import static org.sonar.application.process.EsManagedProcess.Status.YELLOW; public class EsManagedProcess extends AbstractManagedProcess { private static final Logger LOG = LoggerFactory.getLogger(EsManagedProcess.class); private static final int WAIT_FOR_UP_DELAY_IN_MILLIS = 100; - private static final int WAIT_FOR_UP_TIMEOUT = 10 * 60; /* 1min */ + private static final Set SUPPRESSED_ERROR_MESSAGES = ImmutableSet.of("Connection refused", "Timeout connecting"); private volatile boolean nodeOperational = false; + private final int waitForUpTimeout; private final AtomicBoolean firstMasterNotDiscoveredLog = new AtomicBoolean(true); private final EsConnector esConnector; public EsManagedProcess(Process process, ProcessId processId, EsConnector esConnector) { + this(process, processId, esConnector, 10 * 60); + } + + EsManagedProcess(Process process, ProcessId processId, EsConnector esConnector, int waitForUpTimeout) { super(process, processId); this.esConnector = esConnector; + this.waitForUpTimeout = waitForUpTimeout; } @Override @@ -80,7 +89,7 @@ public class EsManagedProcess extends AbstractManagedProcess { i++; status = checkStatus(); } - } while (i < WAIT_FOR_UP_TIMEOUT); + } while (i < waitForUpTimeout); return status == YELLOW || status == GREEN; } @@ -96,6 +105,13 @@ public class EsManagedProcess extends AbstractManagedProcess { } } return KO; + } catch (ElasticsearchException e) { + if (e.status() == RestStatus.INTERNAL_SERVER_ERROR && (SUPPRESSED_ERROR_MESSAGES.stream().anyMatch(s -> e.getMessage().contains(s)))) { + return CONNECTION_REFUSED; + } else { + LOG.error("Failed to check status", e); + } + return KO; } catch (Exception e) { LOG.error("Failed to check status", e); return KO; diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java index d097fa39f53..4bb1b5f92b3 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java @@ -23,9 +23,12 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.AppenderBase; +import java.net.ConnectException; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.concurrent.ExecutionException; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.rest.RestStatus; @@ -41,11 +44,14 @@ import static org.mockito.Mockito.when; public class EsManagedProcessTest { + private final static int WAIT_FOR_UP_TIMEOUT = 1; + private final static int WAIT_FOR_UP_TIMEOUT_LONG = 2; + @Test public void isOperational_should_return_false_if_status_is_unknown() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()).thenReturn(Optional.empty()); - EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isFalse(); } @@ -53,7 +59,7 @@ public class EsManagedProcessTest { public void isOperational_should_return_false_if_Elasticsearch_is_RED() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED)); - EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isFalse(); } @@ -61,7 +67,7 @@ public class EsManagedProcessTest { public void isOperational_should_return_true_if_Elasticsearch_is_YELLOW() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.YELLOW)); - EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isTrue(); } @@ -69,7 +75,7 @@ public class EsManagedProcessTest { public void isOperational_should_return_true_if_Elasticsearch_is_GREEN() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN)); - EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isTrue(); } @@ -77,7 +83,7 @@ public class EsManagedProcessTest { public void isOperational_should_return_true_if_Elasticsearch_was_GREEN_once() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN)); - EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isTrue(); when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED)); @@ -90,7 +96,7 @@ public class EsManagedProcessTest { when(esConnector.getClusterHealthStatus()) .thenReturn(Optional.empty()) .thenReturn(Optional.of(ClusterHealthStatus.GREEN)); - EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isTrue(); } @@ -99,7 +105,34 @@ public class EsManagedProcessTest { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) .thenThrow(new RuntimeException("test")); - EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); + assertThat(underTest.isOperational()).isFalse(); + } + + @Test + public void isOperational_should_return_false_if_ElasticsearchException_with_connection_refused_thrown() { + EsConnector esConnector = mock(EsConnector.class); + when(esConnector.getClusterHealthStatus()) + .thenThrow(new ElasticsearchException("Connection refused")); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); + assertThat(underTest.isOperational()).isFalse(); + } + + @Test + public void isOperational_should_return_false_if_ElasticsearchException_with_connection_timeout_thrown() { + EsConnector esConnector = mock(EsConnector.class); + when(esConnector.getClusterHealthStatus()) + .thenThrow(new ElasticsearchException(new ExecutionException(new ConnectException("Timeout connecting to [/127.0.0.1:9001]")))); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT_LONG); + assertThat(underTest.isOperational()).isFalse(); + } + + @Test + public void isOperational_should_return_false_if_ElasticsearchException_thrown() { + EsConnector esConnector = mock(EsConnector.class); + when(esConnector.getClusterHealthStatus()) + .thenThrow(new ElasticsearchException("test")); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isFalse(); } @@ -116,7 +149,7 @@ public class EsManagedProcessTest { when(esConnector.getClusterHealthStatus()) .thenThrow(new ElasticsearchStatusException("foobar[type=master_not_discovered_exception,acme]...", RestStatus.SERVICE_UNAVAILABLE)); - EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isFalse(); assertThat(memoryAppender.events).isNotEmpty(); assertThat(memoryAppender.events)