SONAR-15418 Suppress Elasticsearch timeout exception when waiting for it to be UP

(cherry picked from commit 4cc250a69e9f592cb6977aa4ce15ed50a83856be)

diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
index bf5bad5d183fee7ccf46bcf851fb59d28a57af8c..356bbbe2b1dbd68bcb1d47a7457961d48263b55f 100644 (file)
--- a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
@@ -19,7 +19,10 @@
  */
 package org.sonar.application.process;
 
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.rest.RestStatus;
@@ -37,15 +40,21 @@ import static org.sonar.application.process.EsManagedProcess.Status.YELLOW;
 public class EsManagedProcess extends AbstractManagedProcess {
   private static final Logger LOG = LoggerFactory.getLogger(EsManagedProcess.class);
   private static final int WAIT_FOR_UP_DELAY_IN_MILLIS = 100;
-  private static final int WAIT_FOR_UP_TIMEOUT = 10 * 60; /* 1min */
+  private static final Set<String> SUPPRESSED_ERROR_MESSAGES = ImmutableSet.of("Connection refused", "Timeout connecting");
 
   private volatile boolean nodeOperational = false;
+  private final int waitForUpTimeout;
   private final AtomicBoolean firstMasterNotDiscoveredLog = new AtomicBoolean(true);
   private final EsConnector esConnector;
 
   public EsManagedProcess(Process process, ProcessId processId, EsConnector esConnector) {
+    this(process, processId, esConnector, 10 * 60);
+  }
+
+  EsManagedProcess(Process process, ProcessId processId, EsConnector esConnector, int waitForUpTimeout) {
     super(process, processId);
     this.esConnector = esConnector;
+    this.waitForUpTimeout = waitForUpTimeout;
   }
 
   @Override
@@ -80,7 +89,7 @@ public class EsManagedProcess extends AbstractManagedProcess {
         i++;
         status = checkStatus();
       }
-    } while (i < WAIT_FOR_UP_TIMEOUT);
+    } while (i < waitForUpTimeout);
     return status == YELLOW || status == GREEN;
   }
 
@@ -96,6 +105,13 @@ public class EsManagedProcess extends AbstractManagedProcess {
         }
       }
       return KO;
+    } catch (ElasticsearchException e) {
+      if (e.status() == RestStatus.INTERNAL_SERVER_ERROR && (SUPPRESSED_ERROR_MESSAGES.stream().anyMatch(s -> e.getMessage().contains(s)))) {
+        return CONNECTION_REFUSED;
+      } else {
+        LOG.error("Failed to check status", e);
+      }
+      return KO;
     } catch (Exception e) {
       LOG.error("Failed to check status", e);
       return KO;

diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
index d097fa39f530a90cdff95245e5c97afae1389387..4bb1b5f92b3a90cc1c33010956b07feb1d535d95 100644 (file)
--- a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
@@ -23,9 +23,12 @@ import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.LoggerContext;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.AppenderBase;
+import java.net.ConnectException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.rest.RestStatus;
@@ -41,11 +44,14 @@ import static org.mockito.Mockito.when;
 
 public class EsManagedProcessTest {
 
+  private final static int WAIT_FOR_UP_TIMEOUT = 1;
+  private final static int WAIT_FOR_UP_TIMEOUT_LONG = 2;
+
   @Test
   public void isOperational_should_return_false_if_status_is_unknown() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.empty());
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isFalse();
   }
 
@@ -53,7 +59,7 @@ public class EsManagedProcessTest {
   public void isOperational_should_return_false_if_Elasticsearch_is_RED() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isFalse();
   }
 
@@ -61,7 +67,7 @@ public class EsManagedProcessTest {
   public void isOperational_should_return_true_if_Elasticsearch_is_YELLOW() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.YELLOW));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isTrue();
   }
 
@@ -69,7 +75,7 @@ public class EsManagedProcessTest {
   public void isOperational_should_return_true_if_Elasticsearch_is_GREEN() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isTrue();
   }
 
@@ -77,7 +83,7 @@ public class EsManagedProcessTest {
   public void isOperational_should_return_true_if_Elasticsearch_was_GREEN_once() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isTrue();
 
     when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED));
@@ -90,7 +96,7 @@ public class EsManagedProcessTest {
     when(esConnector.getClusterHealthStatus())
       .thenReturn(Optional.empty())
       .thenReturn(Optional.of(ClusterHealthStatus.GREEN));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isTrue();
   }
 
@@ -99,7 +105,34 @@ public class EsManagedProcessTest {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus())
       .thenThrow(new RuntimeException("test"));
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
+    assertThat(underTest.isOperational()).isFalse();
+  }
+
+  @Test
+  public void isOperational_should_return_false_if_ElasticsearchException_with_connection_refused_thrown() {
+    EsConnector esConnector = mock(EsConnector.class);
+    when(esConnector.getClusterHealthStatus())
+      .thenThrow(new ElasticsearchException("Connection refused"));
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
+    assertThat(underTest.isOperational()).isFalse();
+  }
+
+  @Test
+  public void isOperational_should_return_false_if_ElasticsearchException_with_connection_timeout_thrown() {
+    EsConnector esConnector = mock(EsConnector.class);
+    when(esConnector.getClusterHealthStatus())
+      .thenThrow(new ElasticsearchException(new ExecutionException(new ConnectException("Timeout connecting to [/127.0.0.1:9001]"))));
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT_LONG);
+    assertThat(underTest.isOperational()).isFalse();
+  }
+
+  @Test
+  public void isOperational_should_return_false_if_ElasticsearchException_thrown() {
+    EsConnector esConnector = mock(EsConnector.class);
+    when(esConnector.getClusterHealthStatus())
+      .thenThrow(new ElasticsearchException("test"));
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isFalse();
   }
 
@@ -116,7 +149,7 @@ public class EsManagedProcessTest {
     when(esConnector.getClusterHealthStatus())
       .thenThrow(new ElasticsearchStatusException("foobar[type=master_not_discovered_exception,acme]...", RestStatus.SERVICE_UNAVAILABLE));
 
-    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT);
     assertThat(underTest.isOperational()).isFalse();
     assertThat(memoryAppender.events).isNotEmpty();
     assertThat(memoryAppender.events)