SONAR-14853 Elasticsearch http encryption

6 files changed, 118 insertions, 22 deletions

diff --git a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java
index 1d81bcab68e..b61aefbe4de 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java
@@ -20,7 +20,10 @@
 package org.sonar.application;
 
 import com.google.common.net.HostAndPort;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import org.sonar.application.cluster.AppNodesClusterHostsConsistency;
@@ -34,6 +37,8 @@ import org.sonar.process.Props;
 import org.sonar.process.cluster.hz.HazelcastMember;
 import org.sonar.process.cluster.hz.HazelcastMemberBuilder;
 
+import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_HTTP_KEYSTORE;
+import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_HTTP_KEYSTORE_PASSWORD;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_HZ_HOSTS;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_KUBERNETES;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HOST;
@@ -78,6 +83,8 @@ public class AppStateFactory {
       .map(HostAndPort::fromString)
       .collect(Collectors.toSet());
     String searchPassword = props.value(CLUSTER_SEARCH_PASSWORD.getKey());
-    return new EsConnectorImpl(hostAndPorts, searchPassword);
+    Path keyStorePath = Optional.ofNullable(props.value(CLUSTER_ES_HTTP_KEYSTORE.getKey())).map(Paths::get).orElse(null);
+    String keyStorePassword = props.value(CLUSTER_ES_HTTP_KEYSTORE_PASSWORD.getKey());
+    return new EsConnectorImpl(hostAndPorts, searchPassword, keyStorePath, keyStorePassword);
   }
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java
index f8c2952070e..06c95efaff9 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java
@@ -54,6 +54,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static java.lang.String.format;
 import static java.util.Collections.singleton;
 import static org.sonar.application.es.EsKeyStoreCli.BOOTSTRAP_PASSWORD_PROPERTY_KEY;
+import static org.sonar.application.es.EsKeyStoreCli.HTTP_KEYSTORE_PASSWORD_PROPERTY_KEY;
 import static org.sonar.application.es.EsKeyStoreCli.KEYSTORE_PASSWORD_PROPERTY_KEY;
 import static org.sonar.application.es.EsKeyStoreCli.TRUSTSTORE_PASSWORD_PROPERTY_KEY;
 import static org.sonar.process.ProcessEntryPoint.PROPERTY_GRACEFUL_STOP_TIMEOUT_MS;
@@ -102,7 +103,8 @@ public class ProcessLauncherImpl implements ProcessLauncher {
       if (processId == ProcessId.ELASTICSEARCH) {
         checkArgument(esInstallation != null, "Incorrect configuration EsInstallation is null");
         EsConnectorImpl esConnector = new EsConnectorImpl(singleton(HostAndPort.fromParts(esInstallation.getHost(),
-          esInstallation.getHttpPort())), esInstallation.getBootstrapPassword());
+          esInstallation.getHttpPort())), esInstallation.getBootstrapPassword(), esInstallation.getHttpKeyStoreLocation(),
+          esInstallation.getHttpKeyStorePassword().orElse(null));
         return new EsManagedProcess(process, processId, esConnector);
       } else {
         ProcessCommands commands = allProcessesCommands.createAfterClean(processId.getIpcIndex());
@@ -140,7 +142,7 @@ public class ProcessLauncherImpl implements ProcessLauncher {
 
     pruneElasticsearchConfDirectory(confDir);
     createElasticsearchConfDirectory(confDir);
-    setupElasticsearchAuthentication(esInstallation);
+    setupElasticsearchSecurity(esInstallation);
 
     esInstallation.getEsYmlSettings().writeToYmlSettingsFile(esInstallation.getElasticsearchYml());
     esInstallation.getEsJvmOptions().writeToJvmOptionFile(esInstallation.getJvmOptions());
@@ -163,26 +165,41 @@ public class ProcessLauncherImpl implements ProcessLauncher {
     }
   }
 
-  private void setupElasticsearchAuthentication(EsInstallation esInstallation) {
+  private void setupElasticsearchSecurity(EsInstallation esInstallation) {
     if (esInstallation.isSecurityEnabled()) {
-      EsKeyStoreCli keyStoreCli = EsKeyStoreCli.getInstance(esInstallation)
-        .store(BOOTSTRAP_PASSWORD_PROPERTY_KEY, esInstallation.getBootstrapPassword());
+      EsKeyStoreCli keyStoreCli = EsKeyStoreCli.getInstance(esInstallation);
 
-      String esConfPath = esInstallation.getConfDirectory().getAbsolutePath();
+      setupElasticsearchAuthentication(esInstallation, keyStoreCli);
+      setupElasticsearchHttpEncryption(esInstallation, keyStoreCli);
 
-      Path trustStoreLocation = esInstallation.getTrustStoreLocation();
-      Path keyStoreLocation = esInstallation.getKeyStoreLocation();
-      if (trustStoreLocation.equals(keyStoreLocation)) {
-        copyFile(trustStoreLocation, Paths.get(esConfPath, trustStoreLocation.toFile().getName()));
-      } else {
-        copyFile(trustStoreLocation, Paths.get(esConfPath, trustStoreLocation.toFile().getName()));
-        copyFile(keyStoreLocation, Paths.get(esConfPath, keyStoreLocation.toFile().getName()));
-      }
+      keyStoreCli.executeWith(this::launchJava);
+    }
+  }
 
-      esInstallation.getTrustStorePassword().ifPresent(s -> keyStoreCli.store(TRUSTSTORE_PASSWORD_PROPERTY_KEY, s));
-      esInstallation.getKeyStorePassword().ifPresent(s -> keyStoreCli.store(KEYSTORE_PASSWORD_PROPERTY_KEY, s));
+  private static void setupElasticsearchAuthentication(EsInstallation esInstallation, EsKeyStoreCli keyStoreCli) {
+    keyStoreCli.store(BOOTSTRAP_PASSWORD_PROPERTY_KEY, esInstallation.getBootstrapPassword());
 
-      keyStoreCli.executeWith(this::launchJava);
+    String esConfPath = esInstallation.getConfDirectory().getAbsolutePath();
+
+    Path trustStoreLocation = esInstallation.getTrustStoreLocation();
+    Path keyStoreLocation = esInstallation.getKeyStoreLocation();
+    if (trustStoreLocation.equals(keyStoreLocation)) {
+      copyFile(trustStoreLocation, Paths.get(esConfPath, trustStoreLocation.toFile().getName()));
+    } else {
+      copyFile(trustStoreLocation, Paths.get(esConfPath, trustStoreLocation.toFile().getName()));
+      copyFile(keyStoreLocation, Paths.get(esConfPath, keyStoreLocation.toFile().getName()));
+    }
+
+    esInstallation.getTrustStorePassword().ifPresent(s -> keyStoreCli.store(TRUSTSTORE_PASSWORD_PROPERTY_KEY, s));
+    esInstallation.getKeyStorePassword().ifPresent(s -> keyStoreCli.store(KEYSTORE_PASSWORD_PROPERTY_KEY, s));
+  }
+
+  private static void setupElasticsearchHttpEncryption(EsInstallation esInstallation, EsKeyStoreCli keyStoreCli) {
+    if (esInstallation.isHttpEncryptionEnabled()) {
+      String esConfPath = esInstallation.getConfDirectory().getAbsolutePath();
+      Path httpKeyStoreLocation = esInstallation.getHttpKeyStoreLocation();
+      copyFile(httpKeyStoreLocation, Paths.get(esConfPath, httpKeyStoreLocation.toFile().getName()));
+      esInstallation.getHttpKeyStorePassword().ifPresent(s -> keyStoreCli.store(HTTP_KEYSTORE_PASSWORD_PROPERTY_KEY, s));
     }
   }
 
diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java
index 9906aeba541..d4fe260aa3a 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java
@@ -21,16 +21,26 @@ package org.sonar.application.es;
 
 import com.google.common.net.HostAndPort;
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
 import java.util.Arrays;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
+import javax.net.ssl.SSLContext;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.client.RequestOptions;
@@ -51,10 +61,15 @@ public class EsConnectorImpl implements EsConnector {
   private final AtomicReference<RestHighLevelClient> restClient = new AtomicReference<>(null);
   private final Set<HostAndPort> hostAndPorts;
   private final String searchPassword;
+  private final Path keyStorePath;
+  private final String keyStorePassword;
 
-  public EsConnectorImpl(Set<HostAndPort> hostAndPorts, @Nullable String searchPassword) {
+  public EsConnectorImpl(Set<HostAndPort> hostAndPorts, @Nullable String searchPassword, @Nullable Path keyStorePath,
+                         @Nullable String keyStorePassword) {
     this.hostAndPorts = hostAndPorts;
     this.searchPassword = searchPassword;
+    this.keyStorePath = keyStorePath;
+    this.keyStorePassword = keyStorePassword;
   }
 
   @Override
@@ -94,7 +109,7 @@ public class EsConnectorImpl implements EsConnector {
 
   private RestHighLevelClient buildRestHighLevelClient() {
     HttpHost[] httpHosts = hostAndPorts.stream()
-      .map(hostAndPort -> new HttpHost(hostAndPort.getHost(), hostAndPort.getPortOrDefault(9001)))
+      .map(this::toHttpHost)
       .toArray(HttpHost[]::new);
 
     if (LOG.isDebugEnabled()) {
@@ -110,15 +125,42 @@ public class EsConnectorImpl implements EsConnector {
           BasicCredentialsProvider provider = getBasicCredentialsProvider(searchPassword);
           httpClientBuilder.setDefaultCredentialsProvider(provider);
         }
+
+        if (keyStorePath != null) {
+          SSLContext sslContext = getSSLContext(keyStorePath, keyStorePassword);
+          httpClientBuilder.setSSLContext(sslContext);
+        }
+
         return httpClientBuilder;
       });
     return new RestHighLevelClient(builder);
   }
 
+  private HttpHost toHttpHost(HostAndPort host) {
+    try {
+      String scheme = keyStorePath != null ? "https" : HttpHost.DEFAULT_SCHEME_NAME;
+      return new HttpHost(InetAddress.getByName(host.getHost()), host.getPortOrDefault(9001), scheme);
+    } catch (UnknownHostException e) {
+      throw new IllegalStateException("Can not resolve host [" + host + "]", e);
+    }
+  }
+
   private static BasicCredentialsProvider getBasicCredentialsProvider(String searchPassword) {
     BasicCredentialsProvider provider = new BasicCredentialsProvider();
     provider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(ES_USERNAME, searchPassword));
     return provider;
   }
 
+  private static SSLContext getSSLContext(Path keyStorePath, @Nullable String keyStorePassword) {
+    try {
+      KeyStore keyStore = KeyStore.getInstance("pkcs12");
+      try (InputStream is = Files.newInputStream(keyStorePath)) {
+        keyStore.load(is, keyStorePassword == null ? null : keyStorePassword.toCharArray());
+      }
+      SSLContextBuilder sslBuilder = SSLContexts.custom().loadTrustMaterial(keyStore, null);
+      return sslBuilder.build();
+    } catch (IOException | GeneralSecurityException e) {
+      throw new IllegalStateException("Failed to setup SSL context on ES client", e);
+    }
+  }
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java
index ebda731f753..68b21afdc22 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java
@@ -34,6 +34,8 @@ import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.process.Props;
 
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED;
+import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_HTTP_KEYSTORE;
+import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_HTTP_KEYSTORE_PASSWORD;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_KEYSTORE;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_KEYSTORE_PASSWORD;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_TRUSTSTORE;
@@ -73,6 +75,10 @@ public class EsInstallation {
   private final String keyStorePassword;
   @Nullable
   private final String trustStorePassword;
+  private final Path httpKeyStoreLocation;
+  @Nullable
+  private final String httpKeyStorePassword;
+  private final boolean httpEncryptionEnabled;
 
   public EsInstallation(Props props) {
     File sqHomeDir = props.nonNullValueAsFile(PATH_HOME.getKey());
@@ -89,6 +95,9 @@ public class EsInstallation {
     this.keyStorePassword = props.value(CLUSTER_ES_KEYSTORE_PASSWORD.getKey());
     this.trustStoreLocation = getPath(props.value(CLUSTER_ES_TRUSTSTORE.getKey()));
     this.trustStorePassword = props.value(CLUSTER_ES_TRUSTSTORE_PASSWORD.getKey());
+    this.httpKeyStoreLocation = getPath(props.value(CLUSTER_ES_HTTP_KEYSTORE.getKey()));
+    this.httpKeyStorePassword = props.value(CLUSTER_ES_HTTP_KEYSTORE_PASSWORD.getKey());
+    this.httpEncryptionEnabled = securityEnabled && httpKeyStoreLocation != null;
   }
 
   private static Path getPath(@Nullable String path) {
@@ -224,4 +233,16 @@ public class EsInstallation {
   public Optional<String> getTrustStorePassword() {
     return Optional.ofNullable(trustStorePassword);
   }
+
+  public Path getHttpKeyStoreLocation() {
+    return httpKeyStoreLocation;
+  }
+
+  public Optional<String> getHttpKeyStorePassword() {
+    return Optional.ofNullable(httpKeyStorePassword);
+  }
+
+  public boolean isHttpEncryptionEnabled() {
+    return httpEncryptionEnabled;
+  }
 }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsKeyStoreCli.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsKeyStoreCli.java
index 74514120567..d9daddef78f 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/es/EsKeyStoreCli.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsKeyStoreCli.java
@@ -41,6 +41,7 @@ public class EsKeyStoreCli {
   public static final String BOOTSTRAP_PASSWORD_PROPERTY_KEY = "bootstrap.password";
   public static final String KEYSTORE_PASSWORD_PROPERTY_KEY = "xpack.security.transport.ssl.keystore.secure_password";
   public static final String TRUSTSTORE_PASSWORD_PROPERTY_KEY = "xpack.security.transport.ssl.truststore.secure_password";
+  public static final String HTTP_KEYSTORE_PASSWORD_PROPERTY_KEY = "xpack.security.http.ssl.keystore.secure_password";
 
   private static final String MAIN_CLASS_NAME = "org.elasticsearch.launcher.CliToolLauncher";
 
diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java
index 843fd1e3585..b438fd88684 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java
@@ -38,6 +38,7 @@ import static java.lang.String.valueOf;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_DISCOVERY_SEED_HOSTS;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_HOSTS;
+import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_HTTP_KEYSTORE;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_KEYSTORE;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_ES_TRUSTSTORE;
 import static org.sonar.process.ProcessProperties.Property.CLUSTER_NAME;
@@ -98,7 +99,7 @@ public class EsSettings {
     configureFileSystem(builder);
     configureNetwork(builder);
     configureCluster(builder);
-    configureAuthentication(builder);
+    configureSecurity(builder);
     configureOthers(builder);
     LOGGER.info("Elasticsearch listening on [HTTP: {}:{}, TCP: {}:{}]",
       builder.get(ES_HTTP_HOST_KEY), builder.get(ES_HTTP_PORT_KEY),
@@ -111,7 +112,7 @@ public class EsSettings {
     builder.put("path.logs", fileSystem.getLogDirectory().getAbsolutePath());
   }
 
-  private void configureAuthentication(Map<String, String> builder) {
+  private void configureSecurity(Map<String, String> builder) {
     if (clusterEnabled && props.value((CLUSTER_SEARCH_PASSWORD.getKey())) != null) {
 
       String clusterESKeystoreFileName = getFileNameFromPathProperty(CLUSTER_ES_KEYSTORE);
@@ -123,6 +124,13 @@ public class EsSettings {
       builder.put("xpack.security.transport.ssl.verification_mode", "certificate");
       builder.put("xpack.security.transport.ssl.keystore.path", clusterESKeystoreFileName);
       builder.put("xpack.security.transport.ssl.truststore.path", clusterESTruststoreFileName);
+
+      if (props.value(CLUSTER_ES_HTTP_KEYSTORE.getKey()) != null) {
+        String clusterESHttpKeystoreFileName = getFileNameFromPathProperty(CLUSTER_ES_HTTP_KEYSTORE);
+
+        builder.put("xpack.security.http.ssl.enabled", Boolean.TRUE.toString());
+        builder.put("xpack.security.http.ssl.keystore.path", clusterESHttpKeystoreFileName);
+      }
     } else {
       builder.put("xpack.security.autoconfiguration.enabled", Boolean.FALSE.toString());
       builder.put("xpack.security.enabled", Boolean.FALSE.toString());