SONAR-15418 Suppress Elasticsearch timeout exception when waiting for it to be UP

2 files changed, 34 insertions, 13 deletions

diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
index 2917485d8df..16ef734ed0b 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.application.process;
 
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
@@ -38,15 +39,21 @@ import static org.sonar.application.process.EsManagedProcess.Status.YELLOW;
 public class EsManagedProcess extends AbstractManagedProcess {
   private static final Logger LOG = LoggerFactory.getLogger(EsManagedProcess.class);
   private static final int WAIT_FOR_UP_DELAY_IN_MILLIS = 100;
-  private static final int WAIT_FOR_UP_TIMEOUT = 10 * 60; /* 1min */
+  private static final Set<String> SUPPRESSED_ERROR_MESSAGES = Set.of("Connection refused", "Timeout connecting");
 
   private volatile boolean nodeOperational = false;
+  private final int waitForUpTimeout;
   private final AtomicBoolean firstMasterNotDiscoveredLog = new AtomicBoolean(true);
   private final EsConnector esConnector;
 
   public EsManagedProcess(Process process, ProcessId processId, EsConnector esConnector) {
+    this(process, processId, esConnector, 10 * 60);
+  }
+
+  EsManagedProcess(Process process, ProcessId processId, EsConnector esConnector, int waitForUpTimeout) {
     super(process, processId);
     this.esConnector = esConnector;
+    this.waitForUpTimeout = waitForUpTimeout;
   }
 
   @Override
@@ -81,7 +88,7 @@ public class EsManagedProcess extends AbstractManagedProcess {
         i++;
         status = checkStatus();
       }
-    } while (i < WAIT_FOR_UP_TIMEOUT);
+    } while (i < waitForUpTimeout);
     return status == YELLOW || status == GREEN;
   }
 
@@ -98,7 +105,7 @@ public class EsManagedProcess extends AbstractManagedProcess {
       }
       return KO;
     } catch (ElasticsearchException e) {
-      if (e.status() == RestStatus.INTERNAL_SERVER_ERROR  && e.getMessage().contains("Connection refused")) {
+      if (e.status() == RestStatus.INTERNAL_SERVER_ERROR && (SUPPRESSED_ERROR_MESSAGES.stream().anyMatch(s -> e.getMessage().contains(s)))) {
         return CONNECTION_REFUSED;
       } else {
         LOG.error("Failed to check status", e);
diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
index 72c475118dc..8709399b176 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
@@ -23,9 +23,11 @@ import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.LoggerContext;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.AppenderBase;
+import java.net.ConnectException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.ExecutionException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
@@ -42,11 +44,14 @@ import static org.mockito.Mockito.when;
 
 public class EsManagedProcessTest {
 
+  private final static int WAIT_FOR_UP_TIMEOUT = 1;
+  private final static int WAIT_FOR_UP_TIMEOUT_LONG = 2;
+
   @Test
   public void isOperational_should_return_false_if_status_is_unknown() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.empty());
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isFalse();
   }
 
@@ -54,7 +59,7 @@ public class EsManagedProcessTest {
   public void isOperational_should_return_false_if_Elasticsearch_is_RED() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isFalse();
   }
 
@@ -62,7 +67,7 @@ public class EsManagedProcessTest {
   public void isOperational_should_return_true_if_Elasticsearch_is_YELLOW() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.YELLOW));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isTrue();
   }
 
@@ -70,7 +75,7 @@ public class EsManagedProcessTest {
   public void isOperational_should_return_true_if_Elasticsearch_is_GREEN() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isTrue();
   }
 
@@ -78,7 +83,7 @@ public class EsManagedProcessTest {
   public void isOperational_should_return_true_if_Elasticsearch_was_GREEN_once() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isTrue();
 
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED));
@@ -91,7 +96,7 @@ public class EsManagedProcessTest {
     when(esConnector.getClusterHealthStatus())
       .thenReturn(Optional.empty())
       .thenReturn(Optional.of(ClusterHealthStatus.GREEN));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isTrue();
   }
 
@@ -100,7 +105,7 @@ public class EsManagedProcessTest {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus())
       .thenThrow(new RuntimeException("test"));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isFalse();
   }
 
@@ -109,7 +114,16 @@ public class EsManagedProcessTest {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus())
       .thenThrow(new ElasticsearchException("Connection refused"));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
+    assertThat(underTest.isOperational()).isFalse();
+  }
+
+  @Test
+  public void isOperational_should_return_false_if_ElasticsearchException_with_connection_timeout_thrown() {
+    EsConnector esConnector = mock(EsConnector.class);
+    when(esConnector.getClusterHealthStatus())
+      .thenThrow(new ElasticsearchException(new ExecutionException(new ConnectException("Timeout connecting to [/127.0.0.1:9001]"))));
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT_LONG);
     assertThat(underTest.isOperational()).isFalse();
   }
 
@@ -118,7 +132,7 @@ public class EsManagedProcessTest {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus())
       .thenThrow(new ElasticsearchException("test"));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isFalse();
   }
 
@@ -135,7 +149,7 @@ public class EsManagedProcessTest {
     when(esConnector.getClusterHealthStatus())
       .thenThrow(new ElasticsearchStatusException("foobar[type=master_not_discovered_exception,acme]...", RestStatus.SERVICE_UNAVAILABLE));
 
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isFalse();
     assertThat(memoryAppender.events).isNotEmpty();
     assertThat(memoryAppender.events)