import java.io.File;
import java.util.Map;
import java.util.Optional;
+import java.util.Properties;
import org.sonar.application.config.AppSettings;
import org.sonar.process.ProcessId;
import org.sonar.process.ProcessProperties;
+import org.sonar.process.logging.LogbackHelper;
import static org.sonar.process.ProcessProperties.HTTPS_PROXY_HOST;
import static org.sonar.process.ProcessProperties.HTTPS_PROXY_PORT;
Map<String, String> settingsMap = new EsSettings(this.settings.getProps()).build();
+ File logDir = new File(settingsMap.get("path.logs"));
EsCommand res = new EsCommand(ProcessId.ELASTICSEARCH)
.setWorkDir(executable.getParentFile().getParentFile())
.setExecutable(executable)
+ .setConfDir(new File(settingsMap.get("path.conf")))
+ .setLog4j2Properties(buildLog4j2Properties(logDir))
.setArguments(this.settings.getProps().rawProperties())
.setClusterName(settingsMap.get("cluster.name"))
.setHost(settingsMap.get("network.host"))
- // TODO add argument to specify log4j configuration file
- // TODO add argument to specify yaml configuration file
.setPort(Integer.valueOf(settingsMap.get("transport.tcp.port")));
settingsMap.forEach((key, value) -> res.addEsOption("-E" + key + "=" + value));
// .addJavaOptions(settings.getProps().nonNullValue(ProcessProperties.SEARCH_JAVA_ADDITIONAL_OPTS));
}
+ private Properties buildLog4j2Properties(File logDir) {
+ // FIXME create a Log4jHelper which shares code with LogbackHelper to build this Properties object + not make LogbackHelper.resolveLevel public + provide correct log format, rolling policy, ...
+ String logLevel = LogbackHelper.resolveLevel(settings.getProps(), "sonar.log.level", "sonar.log.level.es").toString();
+ Properties log4j2Properties = new Properties();
+ log4j2Properties.put("status", "error");
+ log4j2Properties.put("appender.rolling.type", "RollingFile");
+ log4j2Properties.put("appender.rolling.name", "rolling");
+ log4j2Properties.put("appender.rolling.fileName", new File(logDir, "es.log").getAbsolutePath());
+ log4j2Properties.put("appender.rolling.layout.type", "PatternLayout");
+ log4j2Properties.put("appender.rolling.layout.pattern", "[%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n");
+ log4j2Properties.put("appender.rolling.filePattern", "${sys:es.logs}-%d{yyyy-MM-dd}.log");
+ log4j2Properties.put("appender.rolling.policies.type", "Policies");
+ log4j2Properties.put("appender.rolling.policies.time.type", "TimeBasedTriggeringPolicy");
+ log4j2Properties.put("appender.rolling.policies.time.interval", "1");
+ log4j2Properties.put("appender.rolling.policies.time.modulate", "true");
+ log4j2Properties.put("rootLogger.level", logLevel);
+ log4j2Properties.put("rootLogger.appenderRef.rolling.ref", "rolling");
+ return log4j2Properties;
+ }
+
private static String getExecutable() {
if (System.getProperty("os.name").startsWith("Windows")) {
return "elasticsearch/bin/elasticsearch.bat";
import java.io.File;
import java.util.ArrayList;
import java.util.List;
+import java.util.Properties;
import org.sonar.process.ProcessId;
public class EsCommand extends AbstractCommand<EsCommand> {
private File executable;
+ private File confDir;
private String clusterName;
private String host;
private int port;
+ private Properties log4j2Properties;
private List<String> esOptions = new ArrayList<>();
public EsCommand(ProcessId id) {
return this;
}
+ public File getConfDir() {
+ return confDir;
+ }
+
+ public EsCommand setConfDir(File confDir) {
+ this.confDir = confDir;
+ return this;
+ }
+
public String getClusterName() {
return clusterName;
}
return this;
}
+ public Properties getLog4j2Properties() {
+ return log4j2Properties;
+ }
+
+ public EsCommand setLog4j2Properties(Properties log4j2Properties) {
+ this.log4j2Properties = log4j2Properties;
+ return this;
+ }
+
public List<String> getEsOptions() {
return esOptions;
}
private void configureFileSystem(Map<String, String> builder) {
File homeDir = props.nonNullValueAsFile(ProcessProperties.PATH_HOME);
- File dataDir;
- File logDir;
- // data dir
+ builder.put("path.data", buildDataPath(homeDir).getAbsolutePath());
+ builder.put("path.conf", buildConfDir().getAbsolutePath());
+ builder.put("path.logs", buildLogPath(homeDir).getAbsolutePath());
+ }
+
+ private File buildDataPath(File homeDir) {
String dataPath = props.value(ProcessProperties.PATH_DATA);
if (StringUtils.isNotEmpty(dataPath)) {
- dataDir = new File(dataPath, "es");
- } else {
- dataDir = new File(homeDir, "data/es");
+ return new File(dataPath, "es");
}
- builder.put("path.data", dataDir.getAbsolutePath());
+ return new File(homeDir, "data/es");
+ }
- // log dir
+ private File buildLogPath(File homeDir) {
String logPath = props.value(ProcessProperties.PATH_LOGS);
if (StringUtils.isNotEmpty(logPath)) {
- logDir = new File(logPath);
- } else {
- logDir = new File(homeDir, "log");
+ return new File(logPath);
}
- builder.put("path.logs", logDir.getAbsolutePath());
+ return new File(homeDir, "log");
+ }
+
+ private File buildConfDir() {
+ String tempPath = props.value(ProcessProperties.PATH_TEMP);
+ return new File(new File(tempPath, "conf"), "es");
}
private void configureNetwork(Map<String, String> builder) {
import java.util.Map;
import java.util.Properties;
import java.util.function.Supplier;
+import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonar.process.AllProcessesCommands;
public ProcessMonitor launch(EsCommand esCommand) {
Process process = null;
try {
+ writeConfFiles(esCommand);
ProcessBuilder processBuilder = create(esCommand);
LOG.info("Launch process[{}]: {}", esCommand.getProcessId().getKey(), String.join(" ", processBuilder.command()));
}
}
+ private void writeConfFiles(EsCommand esCommand) {
+ File confDir = esCommand.getConfDir();
+ if (!confDir.exists() && !confDir.mkdirs()) {
+ String error = format("Failed to create temporary configuration directory [%s]", confDir.getAbsolutePath());
+ LOG.error(error);
+ throw new IllegalStateException(error);
+ }
+
+ try {
+ IOUtils.copy(getClass().getResourceAsStream("elasticsearch.yml"), new FileOutputStream(new File(confDir, "elasticsearch.yml")));
+ IOUtils.copy(getClass().getResourceAsStream("jvm.options"), new FileOutputStream(new File(confDir, "jvm.options")));
+ esCommand.getLog4j2Properties().store(new FileOutputStream(new File(confDir, "log4j2.properties")), "log42 properties file for ES bundled in SonarQube");
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new IllegalStateException("Failed to write ES configuration files", e);
+ }
+ }
+
@Override
public ProcessMonitor launch(JavaCommand javaCommand) {
Process process = null;
--- /dev/null
+# ======================== Elasticsearch Configuration =========================
+#
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+# Before you set out to tweak and tune the configuration, make sure you
+# understand what are you trying to accomplish and the consequences.
+#
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
+#
+# Please see the documentation for further information on configuration options:
+# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/settings.html>
+#
+# ---------------------------------- Cluster -----------------------------------
+#
+# Use a descriptive name for your cluster:
+#
+#cluster.name: my-application
+#
+# ------------------------------------ Node ------------------------------------
+#
+# Use a descriptive name for the node:
+#
+#node.name: node-1
+#
+# Add custom attributes to the node:
+#
+#node.attr.rack: r1
+#
+# ----------------------------------- Paths ------------------------------------
+#
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+#path.data: /path/to/data
+#
+# Path to log files:
+#
+#path.logs: /path/to/logs
+#
+# ----------------------------------- Memory -----------------------------------
+#
+# Lock the memory on startup:
+#
+#bootstrap.memory_lock: true
+#
+# Make sure that the heap size is set to about half the memory available
+# on the system and that the owner of the process is allowed to use this
+# limit.
+#
+# Elasticsearch performs poorly when the system is swapping the memory.
+#
+# ---------------------------------- Network -----------------------------------
+#
+# Set the bind address to a specific IP (IPv4 or IPv6):
+#
+#network.host: 192.168.0.1
+#
+# Set a custom port for HTTP:
+#
+#http.port: 9200
+#
+# For more information, see the documentation at:
+# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/modules-network.html>
+#
+# --------------------------------- Discovery ----------------------------------
+#
+# Pass an initial list of hosts to perform discovery when new node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
+#
+#discovery.zen.ping.unicast.hosts: ["host1", "host2"]
+#
+# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
+#
+#discovery.zen.minimum_master_nodes: 3
+#
+# For more information, see the documentation at:
+# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/modules-discovery-zen.html>
+#
+# ---------------------------------- Gateway -----------------------------------
+#
+# Block initial recovery after a full cluster restart until N nodes are started:
+#
+#gateway.recover_after_nodes: 3
+#
+# For more information, see the documentation at:
+# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/modules-gateway.html>
+#
+# ---------------------------------- Various -----------------------------------
+#
+# Require explicit names when deleting indices:
+#
+#action.destructive_requires_name: true
--- /dev/null
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms2g
+-Xmx2g
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# disable calls to System#gc
+-XX:+DisableExplicitGC
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# force the server VM
+-server
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# use old-style file permissions on JDK9
+-Djdk.io.permissionsUseCanonicalPath=true
+
+# flags to keep Netty from being unsafe
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+-Dlog4j.skipJansi=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${heap.dump.path}
+
+## GC logging
+
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${loggc}
+
+# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
+# If documents were already indexed with unquoted fields in a previous version
+# of Elasticsearch, some operations may throw errors.
+#
+# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
+# only for migration purposes.
+#-Delasticsearch.json.allow_unquoted_field_names=true
assertThat(generated.get("path.data")).isNotNull();
assertThat(generated.get("path.logs")).isNotNull();
assertThat(generated.get("path.home")).isNull();
+ assertThat(generated.get("path.conf")).isNotNull();
// http is disabled for security reasons
assertThat(generated.get("http.enabled")).isEqualTo("false");
assertThat(settings.get("path.data")).isEqualTo(new File(dataDir, "es").getAbsolutePath());
assertThat(settings.get("path.logs")).isEqualTo(logDir.getAbsolutePath());
assertThat(settings.get("path.home")).isNull();
+ assertThat(settings.get("path.conf")).isEqualTo(new File(tempDir, "conf/es").getAbsolutePath());
}
@Test
*
* @throws IllegalArgumentException if the value of the specified property is not one of {@link #ALLOWED_ROOT_LOG_LEVELS}
*/
- private static Level resolveLevel(Props props, String... propertyKeys) {
+ public static Level resolveLevel(Props props, String... propertyKeys) {
Level newLevel = Level.INFO;
for (String propertyKey : propertyKeys) {
Level level = getPropertyValueAsLevel(props, propertyKey);
projects / sonarqube.git / commitdiff