SONAR-9714 Pass Elasticsearch settings in a yaml file

instead of command line arguments

pom.xml

@@ -805,6 +805,11 @@
         <artifactId>jackson-annotations</artifactId>
         <version>${jackson.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.yaml</groupId>
+        <artifactId>snakeyaml</artifactId>
+        <version>1.15</version>
+      </dependency>
       <dependency>
         <groupId>org.mybatis</groupId>
         <artifactId>mybatis</artifactId>

server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java

@@ -29,7 +29,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.function.Supplier;
-import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.process.ProcessId;
@@ -90,7 +89,7 @@ public class ProcessLauncherImpl implements ProcessLauncher {
     }
   }
 
-  private void writeConfFiles(EsCommand esCommand) {
+  private static void writeConfFiles(EsCommand esCommand) {
     EsFileSystem esFileSystem = esCommand.getFileSystem();
     File confDir = esFileSystem.getConfDirectory();
     if (!confDir.exists() && !confDir.mkdirs()) {
@@ -100,7 +99,7 @@ public class ProcessLauncherImpl implements ProcessLauncher {
     }
 
     try {
-      IOUtils.copy(getClass().getResourceAsStream("elasticsearch.yml"), new FileOutputStream(esFileSystem.getElasticsearchYml()));
+      esCommand.getEsYmlSettings().writeToYmlSettingsFile(esFileSystem.getElasticsearchYml());
       esCommand.getEsJvmOptions().writeToJvmOptionFile(esFileSystem.getJvmOptions());
       esCommand.getLog4j2Properties().store(new FileOutputStream(esFileSystem.getLog4j2Properties()), "log4j2 properties file for ES bundled in SonarQube");
     } catch (IOException e) {

server/sonar-process/pom.xml

@@ -40,6 +40,10 @@
       <groupId>commons-lang</groupId>
       <artifactId>commons-lang</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.yaml</groupId>
+      <artifactId>snakeyaml</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>

server/sonar-process/src/main/java/org/sonar/process/command/CommandFactoryImpl.java

@@ -28,6 +28,7 @@ import org.sonar.process.Props;
 import org.sonar.process.es.EsFileSystem;
 import org.sonar.process.es.EsLogging;
 import org.sonar.process.es.EsSettings;
+import org.sonar.process.es.EsYmlSettings;
 import org.sonar.process.jmvoptions.CeJvmOptions;
 import org.sonar.process.jmvoptions.EsJvmOptions;
 import org.sonar.process.jmvoptions.JvmOptions;
@@ -68,22 +69,20 @@ public class CommandFactoryImpl implements CommandFactory {
     }
     Map<String, String> settingsMap = new EsSettings(props, esFileSystem).build();
 
-    EsCommand res = new EsCommand(ProcessId.ELASTICSEARCH, esFileSystem.getHomeDirectory())
+    return new EsCommand(ProcessId.ELASTICSEARCH, esFileSystem.getHomeDirectory())
       .setFileSystem(esFileSystem)
       .setLog4j2Properties(new EsLogging().createProperties(props, esFileSystem.getLogDirectory()))
       .setArguments(props.rawProperties())
       .setClusterName(settingsMap.get("cluster.name"))
       .setHost(settingsMap.get("network.host"))
       .setPort(Integer.valueOf(settingsMap.get("transport.tcp.port")))
+      .addEsOption("-Epath.conf="+esFileSystem.getConfDirectory().getAbsolutePath())
       .setEsJvmOptions(new EsJvmOptions()
         .addFromMandatoryProperty(props, ProcessProperties.SEARCH_JAVA_OPTS)
         .addFromMandatoryProperty(props, ProcessProperties.SEARCH_JAVA_ADDITIONAL_OPTS))
+      .setEsYmlSettings(new EsYmlSettings(settingsMap))
       .setEnvVariable("ES_JVM_OPTIONS", esFileSystem.getJvmOptions().getAbsolutePath())
       .setEnvVariable("JAVA_HOME", System.getProperties().getProperty("java.home"));
-
-    settingsMap.forEach((key, value) -> res.addEsOption("-E" + key + "=" + value));
-
-    return res;
   }
 
   @Override

server/sonar-process/src/main/java/org/sonar/process/command/EsCommand.java

@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Properties;
 import org.sonar.process.ProcessId;
 import org.sonar.process.es.EsFileSystem;
+import org.sonar.process.es.EsYmlSettings;
 import org.sonar.process.jmvoptions.EsJvmOptions;
 
 public class EsCommand extends AbstractCommand<EsCommand> {
@@ -35,6 +36,7 @@ public class EsCommand extends AbstractCommand<EsCommand> {
   private Properties log4j2Properties;
   private List<String> esOptions = new ArrayList<>();
   private EsJvmOptions esJvmOptions;
+  private EsYmlSettings esYmlSettings;
 
   public EsCommand(ProcessId id, File workDir) {
     super(id, workDir);
@@ -104,4 +106,13 @@ public class EsCommand extends AbstractCommand<EsCommand> {
   public EsJvmOptions getEsJvmOptions() {
     return esJvmOptions;
   }
+
+  public EsCommand setEsYmlSettings(EsYmlSettings esYmlSettings) {
+    this.esYmlSettings = esYmlSettings;
+    return this;
+  }
+
+  public EsYmlSettings getEsYmlSettings() {
+    return esYmlSettings;
+  }
 }

server/sonar-process/src/main/java/org/sonar/process/es/EsYmlSettings.java

@@ -0,0 +1,56 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.process.es;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.util.Map;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+
+import static org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK;
+
+public class EsYmlSettings {
+  private static final String ELASTICSEARCH_YML_OPTIONS_HEADER = "# This file has been automatically generated by SonarQube during startup.\n" +
+    "\n" +
+    "# DO NOT EDIT THIS FILE\n" +
+    "\n";
+
+  private final Map<String, String> elasticsearchSettings;
+
+  public EsYmlSettings(Map<String, String> elasticsearchSettings) {
+    this.elasticsearchSettings = elasticsearchSettings;
+  }
+
+  public void writeToYmlSettingsFile(File file) {
+    DumperOptions dumperOptions = new DumperOptions();
+    dumperOptions.setPrettyFlow(true);
+    dumperOptions.setDefaultFlowStyle(BLOCK);
+    Yaml yaml = new Yaml(dumperOptions);
+    String output = ELASTICSEARCH_YML_OPTIONS_HEADER + yaml.dump(elasticsearchSettings);
+    try {
+      Files.write(file.toPath(), output.getBytes(Charset.forName("UTF-8")));
+    } catch (IOException e) {
+      throw new IllegalStateException("Cannot write Elasticsearch yml settings file", e);
+    }
+  }
+}

tests/src/test/java/org/sonarqube/tests/Category5Suite.java

@@ -28,6 +28,7 @@ import org.sonarqube.tests.rule.RuleEsResilienceTest;
 import org.sonarqube.tests.serverSystem.ClusterTest;
 import org.sonarqube.tests.serverSystem.RestartTest;
 import org.sonarqube.tests.serverSystem.ServerSystemRestartingOrchestrator;
+import org.sonarqube.tests.settings.ElasticsearchSettingsTest;
 import org.sonarqube.tests.settings.LicensesPageTest;
 import org.sonarqube.tests.settings.SettingsTestRestartingOrchestrator;
 import org.sonarqube.tests.telemetry.TelemetryOptOutTest;
@@ -62,7 +63,9 @@ import org.sonarqube.tests.user.UserEsResilienceTest;
   TelemetryUploadTest.class,
   TelemetryOptOutTest.class,
   // ce
-  CeWorkersTest.class
+  CeWorkersTest.class,
+  // elasticsearch
+  ElasticsearchSettingsTest.class
 })
 public class Category5Suite {
 

tests/src/test/java/org/sonarqube/tests/settings/ElasticsearchSettingsTest.java

@@ -0,0 +1,59 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarqube.tests.settings;
+
+import com.sonar.orchestrator.Orchestrator;
+import com.sonar.orchestrator.http.HttpClient;
+import com.sonar.orchestrator.http.HttpResponse;
+import java.net.InetAddress;
+import okhttp3.HttpUrl;
+import org.junit.Test;
+import org.sonar.process.NetworkUtils;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ElasticsearchSettingsTest {
+
+  @Test
+  public void set_http_port_through_sonar_properties() throws Exception {
+    int port = NetworkUtils.getNextAvailablePort(InetAddress.getLoopbackAddress());
+    Orchestrator orchestrator = Orchestrator
+      .builderEnv()
+      .setServerProperty("sonar.search.httpPort", "" + port)
+      .setServerProperty("sonar.search.host", InetAddress.getLoopbackAddress().getHostAddress())
+      .build();
+
+    orchestrator.start();
+
+    try {
+      HttpClient httpClient = new HttpClient.Builder().build();
+      HttpUrl url = new HttpUrl.Builder()
+        .scheme("http")
+        .host(InetAddress.getLoopbackAddress().getHostAddress())
+        .port(port)
+        .addEncodedPathSegments("_cluster/state")
+        .build();
+      HttpResponse response = httpClient.newCall(url).execute();
+      assertThat(response.isSuccessful()).isTrue();
+    } finally {
+      orchestrator.stop();
+    }
+  }
+}