SONAR-12686 upgrade es client to 7.9.3 and move to HTTP

- add should minimum match eq 1 to user index queries
ES 7.X changed behaviour in case filter query with bool it defaults to '0'
https://www.elastic.co/guide/en/elasticsearch/reference/7.x/breaking-changes-7.0.html#_the_filter_context_has_been_removed

- fix issue index routing param
ES 7.X helped discover this bug as new setting has been auto configured which is 'index.number_of_routing_shards'.
This has changed how documents are distributed across shards depending on how many shards the index has.

Without that change issues docs has been incorrectly routed to the same shard hash as projects and it worked no matter what routing key you used projectUuid or auth_projectUuid.

- update ngram and edge_ngram names to match with es 7.x
nGram and edgeNgram has been deprecated in favour of ngram and edge_ngram
https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#deprecated-ngram-edgengram-token-filter-cannot-be-used

- remove `_all : enabled` usage from UT
This field was already deprecated in 6.X, now it has been removed.
https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#all-meta-field-removed

- add Elasticsearch High Level REST client dependency

- use sonar.search.port for ES HTTP

- main process use ES Rest client to check ES status

- sonar.cluster.search.hosts has HTTP ports on APP nodes
also sonar.search.port and sonar.search.host MUST be configured on each Search node with the host and HTTP port of the current node

- use Elasticsearch high level rest client

- use in EsTester

- use as primary es client

- use indices api to get all indices name instead of cluster api

- use cluster health api to check cluster state

- support raw requests for 'nodes/_stats' and '_cluster/stats'

- support raw requests for 'indices/_stats'

- leave netty4plugin as testCompile dependency it is used in UTs

- all ES non-test calls go through EsClient class

- add rest client ES profiling

17 files changed, 273 insertions, 193 deletions

diff --git a/server/sonar-main/build.gradle b/server/sonar-main/build.gradle
index cb67a49fe04..2e11dd66e75 100644
--- a/server/sonar-main/build.gradle
+++ b/server/sonar-main/build.gradle
@@ -15,13 +15,11 @@ dependencies {
   compile 'com.hazelcast:hazelcast'
   compile 'commons-io:commons-io'
   compile 'commons-lang:commons-lang'
-  compile 'io.netty:netty-common'
   compile 'org.apache.logging.log4j:log4j-to-slf4j'
   compile 'org.apache.logging.log4j:log4j-api'
-  compile 'org.elasticsearch.client:transport'
+  compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client'
   compile 'org.elasticsearch:elasticsearch'
   compile 'org.elasticsearch:elasticsearch-core'
-  compile 'org.elasticsearch.plugin:transport-netty4-client'
   compile 'org.slf4j:slf4j-api'
   compile 'org.yaml:snakeyaml'
   
@@ -36,5 +34,7 @@ dependencies {
   testCompile 'org.assertj:assertj-core'
   testCompile 'org.awaitility:awaitility'
   testCompile 'org.mockito:mockito-core'
+  testCompile 'com.squareup.okhttp3:mockwebserver'
+  testCompile 'commons-logging:commons-logging:1.1.1'
   testCompile project(':sonar-testing-harness')
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java
index 6a70a5ce795..b486faadbd8 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.application;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.net.HostAndPort;
 import java.util.Arrays;
 import java.util.Set;
@@ -31,16 +32,18 @@ import org.sonar.application.es.EsConnector;
 import org.sonar.application.es.EsConnectorImpl;
 import org.sonar.process.ProcessId;
 import org.sonar.process.Props;
+import org.sonar.process.cluster.NodeType;
 import org.sonar.process.cluster.hz.HazelcastMember;
 import org.sonar.process.cluster.hz.HazelcastMemberBuilder;
 
 import static java.util.Arrays.asList;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_HZ_HOSTS;
-import static org.sonar.process.ProcessProperties.Property.CLUSTER_NAME;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HOST;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HZ_PORT;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_NAME;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_HOSTS;
+import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST;
+import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT;
 
 public class AppStateFactory {
   private final AppSettings settings;
@@ -70,10 +73,17 @@ public class AppStateFactory {
   }
 
   private static EsConnector createEsConnector(Props props) {
+    NodeType nodeType = ClusterSettings.toNodeType(props);
+    if (nodeType == NodeType.SEARCH) {
+      String host = props.nonNullValue(SEARCH_HOST.getKey());
+      String port = props.nonNullValue(SEARCH_PORT.getKey());
+      return new EsConnectorImpl(ImmutableSet.of(HostAndPort.fromParts(host, Integer.valueOf(port))));
+    }
+
     String searchHosts = props.nonNullValue(CLUSTER_SEARCH_HOSTS.getKey());
     Set<HostAndPort> hostAndPorts = Arrays.stream(searchHosts.split(","))
       .map(HostAndPort::fromString)
       .collect(Collectors.toSet());
-    return new EsConnectorImpl(props.nonNullValue(CLUSTER_NAME.getKey()), hostAndPorts);
+    return new EsConnectorImpl(hostAndPorts);
   }
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java
index e0c4aab97f2..d6a23e5fe2f 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java
@@ -97,7 +97,7 @@ public class ProcessLauncherImpl implements ProcessLauncher {
     try {
       if (processId == ProcessId.ELASTICSEARCH) {
         checkArgument(esInstallation != null, "Incorrect configuration EsInstallation is null");
-        EsConnectorImpl esConnector = new EsConnectorImpl(esInstallation.getClusterName(), singleton(HostAndPort.fromParts(esInstallation.getHost(), esInstallation.getPort())));
+        EsConnectorImpl esConnector = new EsConnectorImpl(singleton(HostAndPort.fromParts(esInstallation.getHost(), esInstallation.getHttpPort())));
         return new EsManagedProcess(process, processId, esConnector);
       } else {
         ProcessCommands commands = allProcessesCommands.createAfterClean(processId.getIpcIndex());
diff --git a/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java b/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java
index 5efe6f0b999..6eec7ff5746 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java
@@ -200,8 +200,9 @@ public class ClusterAppStateImpl implements ClusterAppState {
   }
 
   private boolean isElasticSearchAvailable() {
-    ClusterHealthStatus clusterHealthStatus = esConnector.getClusterHealthStatus();
-    return clusterHealthStatus.equals(ClusterHealthStatus.GREEN) || clusterHealthStatus.equals(ClusterHealthStatus.YELLOW);
+    return esConnector.getClusterHealthStatus()
+      .filter(t -> ClusterHealthStatus.GREEN.equals(t) || ClusterHealthStatus.YELLOW.equals(t))
+      .isPresent();
   }
 
   private class OperationalProcessListener implements EntryListener<ClusterProcess, Boolean> {
diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java
index 37cd0c6c31e..353780521d3 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java
@@ -143,9 +143,8 @@ public class CommandFactoryImpl implements CommandFactory {
         .addFromMandatoryProperty(props, SEARCH_JAVA_OPTS.getKey())
         .addFromMandatoryProperty(props, SEARCH_JAVA_ADDITIONAL_OPTS.getKey()))
       .setEsYmlSettings(new EsYmlSettings(settingsMap))
-      .setClusterName(settingsMap.get("cluster.name"))
-      .setHost(settingsMap.get("network.host"))
-      .setPort(Integer.parseInt(settingsMap.get("transport.port")));
+      .setHost(settingsMap.get("http.host"))
+      .setHttpPort(Integer.parseInt(settingsMap.get("http.port")));
     return esInstallation;
   }
 
diff --git a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java
index 85d7078381c..997e68a5f50 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java
@@ -138,7 +138,7 @@ public class ClusterSettings implements Consumer<Props> {
     return addressAndPort;
   }
 
-  private static NodeType toNodeType(Props props) {
+  public static NodeType toNodeType(Props props) {
     String nodeTypeValue = requireValue(props, CLUSTER_NODE_TYPE);
     if (!NodeType.isValid(nodeTypeValue)) {
       throw new MessageException(format("Invalid value for property %s: [%s], only [%s] are allowed", CLUSTER_NODE_TYPE.getKey(), nodeTypeValue,
diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java
index 6e289407904..78544b302a5 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java
@@ -19,9 +19,11 @@
  */
 package org.sonar.application.es;
 
+import java.util.Optional;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 
 public interface EsConnector {
-  ClusterHealthStatus getClusterHealthStatus();
+  Optional<ClusterHealthStatus> getClusterHealthStatus();
+
   void stop();
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java
index 8252014f12d..bacc25c85b6 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java
@@ -20,121 +20,83 @@
 package org.sonar.application.es;
 
 import com.google.common.net.HostAndPort;
-import io.netty.util.ThreadDeathWatcher;
-import io.netty.util.concurrent.GlobalEventExecutor;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
+import org.apache.http.HttpHost;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.transport.Netty4Plugin;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static java.lang.String.format;
-import static java.util.Collections.singletonList;
-import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 
 public class EsConnectorImpl implements EsConnector {
 
   private static final Logger LOG = LoggerFactory.getLogger(EsConnectorImpl.class);
 
-  private final AtomicReference<TransportClient> transportClient = new AtomicReference<>(null);
-  private final String clusterName;
+  private final AtomicReference<RestHighLevelClient> restClient = new AtomicReference<>(null);
   private final Set<HostAndPort> hostAndPorts;
 
-  public EsConnectorImpl(String clusterName, Set<HostAndPort> hostAndPorts) {
-    this.clusterName = clusterName;
+  public EsConnectorImpl(Set<HostAndPort> hostAndPorts) {
     this.hostAndPorts = hostAndPorts;
   }
 
   @Override
-  public ClusterHealthStatus getClusterHealthStatus() {
-    return getTransportClient().admin().cluster()
-      .health(new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.YELLOW).timeout(timeValueSeconds(30)))
-      .actionGet().getStatus();
+  public Optional<ClusterHealthStatus> getClusterHealthStatus() {
+    try {
+      ClusterHealthResponse healthResponse = getRestHighLevelClient().cluster()
+        .health(new ClusterHealthRequest().waitForYellowStatus().timeout(timeValueSeconds(30)), RequestOptions.DEFAULT);
+      return Optional.of(healthResponse.getStatus());
+    } catch (IOException e) {
+      LOG.trace("Failed to check health status ", e);
+      return Optional.empty();
+    }
   }
 
   @Override
   public void stop() {
-    transportClient.set(null);
-  }
-
-  private TransportClient getTransportClient() {
-    TransportClient res = this.transportClient.get();
-    if (res == null) {
-      res = buildTransportClient();
-      if (this.transportClient.compareAndSet(null, res)) {
-        return res;
+    RestHighLevelClient restHighLevelClient = restClient.get();
+    if (restHighLevelClient != null) {
+      try {
+        restHighLevelClient.close();
+      } catch (IOException e) {
+        LOG.warn("Error occurred while closing Rest Client", e);
       }
-      return this.transportClient.get();
     }
-    return res;
   }
 
-  private TransportClient buildTransportClient() {
-    Settings.Builder esSettings = Settings.builder();
-
-    // mandatory property defined by bootstrap process
-    esSettings.put("cluster.name", clusterName);
-
-    TransportClient nativeClient = new MinimalTransportClient(esSettings.build(), hostAndPorts);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Connected to Elasticsearch node: [{}]", displayedAddresses(nativeClient));
+  private RestHighLevelClient getRestHighLevelClient() {
+    RestHighLevelClient res = this.restClient.get();
+    if (res != null) {
+      return res;
     }
-    return nativeClient;
-  }
 
-  private static String displayedAddresses(TransportClient nativeClient) {
-    return nativeClient.transportAddresses().stream().map(TransportAddress::toString).collect(Collectors.joining(", "));
+    RestHighLevelClient restHighLevelClient = buildRestHighLevelClient();
+    this.restClient.set(restHighLevelClient);
+    return restHighLevelClient;
   }
 
-  private static class MinimalTransportClient extends TransportClient {
-
-    public MinimalTransportClient(Settings settings, Set<HostAndPort> hostAndPorts) {
-      super(settings, unmodifiableList(singletonList(Netty4Plugin.class)));
+  private RestHighLevelClient buildRestHighLevelClient() {
+    HttpHost[] httpHosts = hostAndPorts.stream()
+      .map(hostAndPort -> new HttpHost(hostAndPort.getHost(), hostAndPort.getPortOrDefault(9001)))
+      .toArray(HttpHost[]::new);
 
-      boolean connectedToOneHost = false;
-      for (HostAndPort hostAndPort : hostAndPorts) {
-        try {
-          addTransportAddress(new TransportAddress(InetAddress.getByName(hostAndPort.getHost()), hostAndPort.getPortOrDefault(9001)));
-          connectedToOneHost = true;
-        } catch (UnknownHostException e) {
-          LOG.debug("Can not resolve host [" + hostAndPort.getHost() + "]", e);
-        }
-      }
-      if (!connectedToOneHost) {
-        throw new IllegalStateException(format("Can not connect to one node from [%s]",
-          hostAndPorts.stream()
-            .map(h -> format("%s:%d", h.getHost(), h.getPortOrDefault(9001)))
-            .collect(Collectors.joining(","))));
-      }
-    }
-
-    @Override
-    public void close() {
-      super.close();
-      if (!NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings)
-        || NetworkModule.TRANSPORT_TYPE_SETTING.get(settings).equals(Netty4Plugin.NETTY_TRANSPORT_NAME)) {
-        try {
-          GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
-        }
-        try {
-          ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
-        }
-      }
+    if (LOG.isDebugEnabled()) {
+      String addresses = Arrays.stream(httpHosts)
+        .map(t -> t.getHostName() + ":" + t.getPort())
+        .collect(Collectors.joining(", "));
+      LOG.debug("Connected to Elasticsearch node: [{}]", addresses);
     }
+    return new RestHighLevelClient(RestClient.builder(httpHosts));
   }
+
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java
index 6d084d23cba..5657f094adb 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java
@@ -52,7 +52,7 @@ public class EsInstallation {
   private Properties log4j2Properties;
   private String clusterName;
   private String host;
-  private int port;
+  private int httpPort;
 
   public EsInstallation(Props props) {
     File sqHomeDir = props.nonNullValueAsFile(PATH_HOME.getKey());
@@ -152,15 +152,6 @@ public class EsInstallation {
     return this;
   }
 
-  public String getClusterName() {
-    return clusterName;
-  }
-
-  public EsInstallation setClusterName(String clusterName) {
-    this.clusterName = clusterName;
-    return this;
-  }
-
   public String getHost() {
     return host;
   }
@@ -170,12 +161,12 @@ public class EsInstallation {
     return this;
   }
 
-  public int getPort() {
-    return port;
+  public int getHttpPort() {
+    return httpPort;
   }
 
-  public EsInstallation setPort(int port) {
-    this.port = port;
+  public EsInstallation setHttpPort(int httpPort) {
+    this.httpPort = httpPort;
     return this;
   }
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java
index a852fdf4efc..1a7b658c70a 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java
@@ -36,9 +36,9 @@ import static org.sonar.process.ProcessProperties.Property.CLUSTER_NAME;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_NAME;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_HOSTS;
 import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST;
-import static org.sonar.process.ProcessProperties.Property.SEARCH_HTTP_PORT;
 import static org.sonar.process.ProcessProperties.Property.SEARCH_INITIAL_STATE_TIMEOUT;
 import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT;
+import static org.sonar.process.ProcessProperties.Property.SEARCH_TRANSPORT_PORT;
 
 public class EsSettings {
 
@@ -75,8 +75,8 @@ public class EsSettings {
   public Map<String, String> build() {
     Map<String, String> builder = new HashMap<>();
     configureFileSystem(builder);
-    configureNetwork(builder);
-    configureCluster(builder);
+    InetAddress host = configureNetwork(builder);
+    configureCluster(builder, host);
     configureOthers(builder);
     return builder;
   }
@@ -86,33 +86,27 @@ public class EsSettings {
     builder.put("path.logs", fileSystem.getLogDirectory().getAbsolutePath());
   }
 
-  private void configureNetwork(Map<String, String> builder) {
+  private InetAddress configureNetwork(Map<String, String> builder) {
     InetAddress host = readHost();
-    int port = Integer.parseInt(props.nonNullValue(SEARCH_PORT.getKey()));
-    LOGGER.info("Elasticsearch listening on {}:{}", host, port);
+    int httpPort = Integer.parseInt(props.nonNullValue(SEARCH_PORT.getKey()));
+    LOGGER.info("Elasticsearch listening on {}:{}", host, httpPort);
 
-    // FIXME no need to open TCP port unless running DCE
-    //       TCP is used by main process to check ES is up => probably has to use HTTP now
-    builder.put("transport.port", valueOf(port));
-    builder.put("transport.host", valueOf(host.getHostAddress()));
-    builder.put("network.host", valueOf(host.getHostAddress()));
+    // see https://github.com/lmenezes/elasticsearch-kopf/issues/195
+    builder.put("http.cors.enabled", valueOf(true));
+    builder.put("http.cors.allow-origin", "*");
+    builder.put("http.host", host.getHostAddress());
+    builder.put("http.port", valueOf(httpPort));
 
+    builder.put("network.host", valueOf(host.getHostAddress()));
     // Elasticsearch sets the default value of TCP reuse address to true only on non-MSWindows machines, but why ?
     builder.put("network.tcp.reuse_address", valueOf(true));
 
-    int httpPort = props.valueAsInt(SEARCH_HTTP_PORT.getKey(), -1);
-    if (httpPort < 0) {
-      // standard configuration
-      builder.put("http.enabled", valueOf(false));
-    } else {
-      LOGGER.warn("Elasticsearch HTTP connector is enabled on port {}. MUST NOT BE USED FOR PRODUCTION", httpPort);
-      // see https://github.com/lmenezes/elasticsearch-kopf/issues/195
-      builder.put("http.cors.enabled", valueOf(true));
-      builder.put("http.cors.allow-origin", "*");
-      builder.put("http.enabled", valueOf(true));
-      builder.put("http.host", host.getHostAddress());
-      builder.put("http.port", valueOf(httpPort));
-    }
+    // FIXME remove definition of transport properties when Web and CE have moved to ES Rest client
+    int tcpPort = props.valueAsInt(SEARCH_TRANSPORT_PORT.getKey(), 9002);
+    builder.put("transport.port", valueOf(tcpPort));
+    builder.put("transport.host", valueOf(host.getHostAddress()));
+
+    return host;
   }
 
   private InetAddress readHost() {
@@ -124,7 +118,7 @@ public class EsSettings {
     }
   }
 
-  private void configureCluster(Map<String, String> builder) {
+  private void configureCluster(Map<String, String> builder, InetAddress host) {
     // Default value in a standalone mode, not overridable
 
     String initialStateTimeOut = "30s";
@@ -132,6 +126,10 @@ public class EsSettings {
     if (clusterEnabled) {
       initialStateTimeOut = props.value(SEARCH_INITIAL_STATE_TIMEOUT.getKey(), "120s");
 
+      int tcpPort = props.valueAsInt(SEARCH_TRANSPORT_PORT.getKey(), 9002);
+      builder.put("transport.port", valueOf(tcpPort));
+      builder.put("transport.host", valueOf(host.getHostAddress()));
+
       String hosts = props.value(CLUSTER_SEARCH_HOSTS.getKey(), "");
       LOGGER.info("Elasticsearch cluster enabled. Connect to hosts [{}]", hosts);
       builder.put("discovery.seed_hosts", hosts);
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
index 6fe8f1acc35..36008dc3988 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
@@ -20,8 +20,9 @@
 package org.sonar.application.process;
 
 import java.util.concurrent.atomic.AtomicBoolean;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.rest.RestStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.application.es.EsConnector;
@@ -85,21 +86,14 @@ public class EsManagedProcess extends AbstractManagedProcess {
 
   private Status checkStatus() {
     try {
-      switch (esConnector.getClusterHealthStatus()) {
-        case GREEN:
-          return GREEN;
-        case YELLOW:
-          return YELLOW;
-        case RED:
-          return RED;
-        default:
-          return KO;
-      }
-    } catch (NoNodeAvailableException e) {
-      return CONNECTION_REFUSED;
-    } catch (MasterNotDiscoveredException e) {
-      if (firstMasterNotDiscoveredLog.getAndSet(false)) {
-        LOG.info("Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?");
+      return esConnector.getClusterHealthStatus()
+        .map(EsManagedProcess::convert)
+        .orElse(CONNECTION_REFUSED);
+    } catch (ElasticsearchStatusException e) {
+      if (e.status() == RestStatus.SERVICE_UNAVAILABLE && e.getMessage().contains("type=master_not_discovered_exception")) {
+        if (firstMasterNotDiscoveredLog.getAndSet(false)) {
+          LOG.info("Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?");
+        }
       }
       return KO;
     } catch (Exception e) {
@@ -108,6 +102,19 @@ public class EsManagedProcess extends AbstractManagedProcess {
     }
   }
 
+  private static Status convert(ClusterHealthStatus clusterHealthStatus) {
+    switch (clusterHealthStatus) {
+      case GREEN:
+        return GREEN;
+      case YELLOW:
+        return YELLOW;
+      case RED:
+        return RED;
+      default:
+        return KO;
+    }
+  }
+
   enum Status {
     CONNECTION_REFUSED, KO, RED, YELLOW, GREEN
   }
diff --git a/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java b/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java
index 1ae9c798b9a..48cbf0754e9 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java
@@ -51,7 +51,7 @@ public class AppStateFactoryTest {
     settings.set(CLUSTER_NODE_HOST.getKey(), ip.get().getHostAddress());
     settings.set(CLUSTER_HZ_HOSTS.getKey(), ip.get().getHostAddress());
     settings.set(CLUSTER_NAME.getKey(), "foo");
-    settings.set(CLUSTER_SEARCH_HOSTS.getKey(), "localhost:9001");
+    settings.set(CLUSTER_SEARCH_HOSTS.getKey(), "localhost:9002");
 
     AppState appState = underTest.create();
     assertThat(appState).isInstanceOf(ClusterAppStateImpl.class);
diff --git a/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java
index 30360924a9a..9612aff58d5 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java
@@ -200,8 +200,7 @@ public class ProcessLauncherImplTest {
       .setEsJvmOptions(mock(EsJvmOptions.class))
       .setLog4j2Properties(new Properties())
       .setHost("localhost")
-      .setPort(9001)
-      .setClusterName("sonarqube"));
+      .setHttpPort(9001));
     return command;
   }
 
@@ -212,8 +211,7 @@ public class ProcessLauncherImplTest {
       .set("sonar.path.data", this.temp.newFolder("data").getAbsolutePath())
       .set("sonar.path.temp", tempFolder.getAbsolutePath())
       .set("sonar.path.logs", this.temp.newFolder("logs").getAbsolutePath()))
-        .setClusterName("cluster")
-        .setPort(9001)
+        .setHttpPort(9001)
         .setHost("localhost")
         .setEsYmlSettings(new EsYmlSettings(new HashMap<>()))
         .setEsJvmOptions(new EsJvmOptions(new Props(new Properties()), tempFolder))
diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java
index ee0537edc3a..bbe96d163b0 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java
@@ -131,9 +131,8 @@ public class CommandFactoryImplTest {
     EsScriptCommand esCommand = (EsScriptCommand) command;
     EsInstallation esConfig = esCommand.getEsInstallation();
 
-    assertThat(esConfig.getClusterName()).isEqualTo("sonarqube");
     assertThat(esConfig.getHost()).isNotEmpty();
-    assertThat(esConfig.getPort()).isEqualTo(9001);
+    assertThat(esConfig.getHttpPort()).isEqualTo(9001);
     assertThat(esConfig.getEsJvmOptions().getAll())
       // enforced values
       .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8")
@@ -167,9 +166,8 @@ public class CommandFactoryImplTest {
     JavaCommand<?> esCommand = (JavaCommand<?>) command;
     EsInstallation esConfig = esCommand.getEsInstallation();
 
-    assertThat(esConfig.getClusterName()).isEqualTo("sonarqube");
     assertThat(esConfig.getHost()).isNotEmpty();
-    assertThat(esConfig.getPort()).isEqualTo(9001);
+    assertThat(esConfig.getHttpPort()).isEqualTo(9001);
     assertThat(esConfig.getEsJvmOptions().getAll())
       // enforced values
       .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8")
@@ -209,8 +207,7 @@ public class CommandFactoryImplTest {
     AbstractCommand esCommand = newFactory(props).createEsCommand();
     EsInstallation esConfig = esCommand.getEsInstallation();
 
-    assertThat(esConfig.getClusterName()).isEqualTo("foo");
-    assertThat(esConfig.getPort()).isEqualTo(1234);
+    assertThat(esConfig.getHttpPort()).isEqualTo(1234);
     assertThat(esConfig.getEsJvmOptions().getAll())
       // enforced values
       .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8")
diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java
new file mode 100644
index 00000000000..3bacc724278
--- /dev/null
+++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java
@@ -0,0 +1,92 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application.es;
+
+import com.google.common.collect.Sets;
+import com.google.common.net.HostAndPort;
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.junit.After;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class EsConnectorImplTest {
+
+  private static final String JSON_SUCCESS_RESPONSE = "{" +
+    "  \"cluster_name\" : \"testcluster\"," +
+    "  \"status\" : \"yellow\"," +
+    "  \"timed_out\" : false," +
+    "  \"number_of_nodes\" : 1," +
+    "  \"number_of_data_nodes\" : 1," +
+    "  \"active_primary_shards\" : 1," +
+    "  \"active_shards\" : 1," +
+    "  \"relocating_shards\" : 0," +
+    "  \"initializing_shards\" : 0," +
+    "  \"unassigned_shards\" : 1," +
+    "  \"delayed_unassigned_shards\": 0," +
+    "  \"number_of_pending_tasks\" : 0," +
+    "  \"number_of_in_flight_fetch\": 0," +
+    "  \"task_max_waiting_in_queue_millis\": 0," +
+    "  \"active_shards_percent_as_number\": 50.0" +
+    "}";
+
+  private static final String JSON_ERROR_RESPONSE = "{" +
+    "  \"error\" : \"i-have-a-bad-feelings-about-this\"" +
+    "}";
+
+  @Rule
+  public MockWebServer mockWebServer = new MockWebServer();
+
+  EsConnectorImpl underTest = new EsConnectorImpl(Sets.newHashSet(HostAndPort.fromParts(mockWebServer.getHostName(), mockWebServer.getPort())));
+
+  @After
+  public void after() {
+    underTest.stop();
+  }
+
+  @Test
+  public void should_rethrow_if_es_exception() {
+    mockServerResponse(500, JSON_ERROR_RESPONSE);
+
+    assertThatThrownBy(() -> underTest.getClusterHealthStatus())
+      .isInstanceOf(ElasticsearchStatusException.class);
+  }
+
+  @Test
+  public void should_return_status() {
+    mockServerResponse(200, JSON_SUCCESS_RESPONSE);
+
+    assertThat(underTest.getClusterHealthStatus())
+      .hasValue(ClusterHealthStatus.YELLOW);
+  }
+
+  private void mockServerResponse(int httpCode, String jsonResponse) {
+    mockWebServer.enqueue(new MockResponse()
+      .setResponseCode(httpCode)
+      .setBody(jsonResponse)
+      .setHeader("Content-Type", "application/json"));
+  }
+
+}
diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java
index cfc7800d52e..306f6a79332 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java
@@ -56,7 +56,6 @@ import static org.sonar.process.ProcessProperties.Property.PATH_HOME;
 import static org.sonar.process.ProcessProperties.Property.PATH_LOGS;
 import static org.sonar.process.ProcessProperties.Property.PATH_TEMP;
 import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST;
-import static org.sonar.process.ProcessProperties.Property.SEARCH_HTTP_PORT;
 import static org.sonar.process.ProcessProperties.Property.SEARCH_INITIAL_STATE_TIMEOUT;
 import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT;
 
@@ -138,9 +137,14 @@ public class EsSettingsTest {
     EsSettings esSettings = new EsSettings(props, new EsInstallation(props), system);
 
     Map<String, String> generated = esSettings.build();
-    assertThat(generated.get("transport.port")).isEqualTo("1234");
+
+    // FIXME transport.port and transport.host should not be set in standalone
+    assertThat(generated.get("transport.port")).isEqualTo("9002");
     assertThat(generated.get("transport.host")).isEqualTo("127.0.0.1");
 
+    assertThat(generated.get("http.port")).isEqualTo("1234");
+    assertThat(generated.get("http.host")).isEqualTo("127.0.0.1");
+
     // no cluster, but cluster and node names are set though
     assertThat(generated.get("cluster.name")).isEqualTo("sonarqube");
     assertThat(generated.get("node.name")).isEqualTo("sonarqube");
@@ -150,9 +154,6 @@ public class EsSettingsTest {
     assertThat(generated.get("path.home")).isNull();
     assertThat(generated.get("path.conf")).isNull();
 
-    // http is disabled for security reasons
-    assertThat(generated.get("http.enabled")).isEqualTo("false");
-
     assertThat(generated.get("discovery.seed_hosts")).isNull();
     assertThat(generated.get("discovery.initial_state_timeout")).isEqualTo("30s");
 
@@ -274,39 +275,51 @@ public class EsSettingsTest {
   }
 
   @Test
-  public void enable_http_connector() throws Exception {
-    Props props = minProps(CLUSTER_DISABLED);
-    props.set(SEARCH_HTTP_PORT.getKey(), "9010");
+  @UseDataProvider("clusterEnabledOrNot")
+  public void enable_http_connector_on_port_9001_by_default(boolean clusterEnabled) throws Exception {
+    Props props = minProps(clusterEnabled);
     Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build();
 
-    assertThat(settings.get("http.port")).isEqualTo("9010");
+    assertThat(settings.get("http.port")).isEqualTo("9001");
     assertThat(settings.get("http.host")).isEqualTo("127.0.0.1");
-    assertThat(settings.get("http.enabled")).isEqualTo("true");
   }
 
   @Test
-  public void enable_http_connector_different_host() throws Exception {
-    Props props = minProps(CLUSTER_DISABLED);
-    props.set(SEARCH_HTTP_PORT.getKey(), "9010");
+  @UseDataProvider("clusterEnabledOrNot")
+  public void enable_http_connector_on_specified_port(boolean clusterEnabled) throws Exception {
+    String port = "" + new Random().nextInt(49151);
+    Props props = minProps(clusterEnabled);
+    props.set(SEARCH_PORT.getKey(), port);
+    Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build();
+
+    assertThat(settings.get("http.port")).isEqualTo(port);
+    assertThat(settings.get("http.host")).isEqualTo("127.0.0.1");
+  }
+
+  @Test
+  @UseDataProvider("clusterEnabledOrNot")
+  public void enable_http_connector_different_host(boolean clusterEnabled) throws Exception {
+    Props props = minProps(clusterEnabled);
     props.set(SEARCH_HOST.getKey(), "127.0.0.2");
     Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build();
 
-    assertThat(settings.get("http.port")).isEqualTo("9010");
+    assertThat(settings.get("http.port")).isEqualTo("9001");
     assertThat(settings.get("http.host")).isEqualTo("127.0.0.2");
-    assertThat(settings.get("http.enabled")).isEqualTo("true");
   }
 
   @Test
-  public void enable_seccomp_filter_by_default() throws Exception {
-    Props props = minProps(CLUSTER_DISABLED);
+  @UseDataProvider("clusterEnabledOrNot")
+  public void enable_seccomp_filter_by_default(boolean clusterEnabled) throws Exception {
+    Props props = minProps(clusterEnabled);
     Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build();
 
     assertThat(settings.get("bootstrap.system_call_filter")).isNull();
   }
 
   @Test
-  public void disable_seccomp_filter_if_configured_in_search_additional_props() throws Exception {
-    Props props = minProps(CLUSTER_DISABLED);
+  @UseDataProvider("clusterEnabledOrNot")
+  public void disable_seccomp_filter_if_configured_in_search_additional_props(boolean clusterEnabled) throws Exception {
+    Props props = minProps(clusterEnabled);
     props.set("sonar.search.javaAdditionalOpts", "-Xmx1G -Dbootstrap.system_call_filter=false -Dfoo=bar");
     Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build();
 
@@ -314,8 +327,9 @@ public class EsSettingsTest {
   }
 
   @Test
-  public void disable_mmap_if_configured_in_search_additional_props() throws Exception {
-    Props props = minProps(CLUSTER_DISABLED);
+  @UseDataProvider("clusterEnabledOrNot")
+  public void disable_mmap_if_configured_in_search_additional_props(boolean clusterEnabled) throws Exception {
+    Props props = minProps(clusterEnabled);
     props.set("sonar.search.javaAdditionalOpts", "-Dnode.store.allow_mmap=false");
     Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build();
 
diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
index 967b319e8b6..d4654af2477 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
@@ -25,9 +25,10 @@ import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.AppenderBase;
 import java.util.ArrayList;
 import java.util.List;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
+import java.util.Optional;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.rest.RestStatus;
 import org.junit.Test;
 import org.slf4j.LoggerFactory;
 import org.sonar.application.es.EsConnector;
@@ -41,9 +42,17 @@ import static org.mockito.Mockito.when;
 public class EsManagedProcessTest {
 
   @Test
+  public void isOperational_should_return_false_if_status_is_unknown() {
+    EsConnector esConnector = mock(EsConnector.class);
+    when(esConnector.getClusterHealthStatus()).thenReturn(Optional.empty());
+    EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
+    assertThat(underTest.isOperational()).isFalse();
+  }
+
+  @Test
   public void isOperational_should_return_false_if_Elasticsearch_is_RED() {
     EsConnector esConnector = mock(EsConnector.class);
-    when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.RED);
+    when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED));
     EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
     assertThat(underTest.isOperational()).isFalse();
   }
@@ -51,7 +60,7 @@ public class EsManagedProcessTest {
   @Test
   public void isOperational_should_return_true_if_Elasticsearch_is_YELLOW() {
     EsConnector esConnector = mock(EsConnector.class);
-    when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.YELLOW);
+    when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.YELLOW));
     EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
     assertThat(underTest.isOperational()).isTrue();
   }
@@ -59,7 +68,7 @@ public class EsManagedProcessTest {
   @Test
   public void isOperational_should_return_true_if_Elasticsearch_is_GREEN() {
     EsConnector esConnector = mock(EsConnector.class);
-    when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.GREEN);
+    when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN));
     EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
     assertThat(underTest.isOperational()).isTrue();
   }
@@ -67,11 +76,11 @@ public class EsManagedProcessTest {
   @Test
   public void isOperational_should_return_true_if_Elasticsearch_was_GREEN_once() {
     EsConnector esConnector = mock(EsConnector.class);
-    when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.GREEN);
+    when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN));
     EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
     assertThat(underTest.isOperational()).isTrue();
 
-    when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.RED);
+    when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED));
     assertThat(underTest.isOperational()).isTrue();
   }
 
@@ -79,8 +88,8 @@ public class EsManagedProcessTest {
   public void isOperational_should_retry_if_Elasticsearch_is_unreachable() {
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus())
-      .thenThrow(new NoNodeAvailableException("test"))
-      .thenReturn(ClusterHealthStatus.GREEN);
+      .thenReturn(Optional.empty())
+      .thenReturn(Optional.of(ClusterHealthStatus.GREEN));
     EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
     assertThat(underTest.isOperational()).isTrue();
   }
@@ -105,7 +114,7 @@ public class EsManagedProcessTest {
 
     EsConnector esConnector = mock(EsConnector.class);
     when(esConnector.getClusterHealthStatus())
-      .thenThrow(new MasterNotDiscoveredException("Master not elected -test-"));
+      .thenThrow(new ElasticsearchStatusException("foobar[type=master_not_discovered_exception,acme]...", RestStatus.SERVICE_UNAVAILABLE));
 
     EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector);
     assertThat(underTest.isOperational()).isFalse();