From 8edde8c99147975908e6bfe3db1a5ba46039b2e4 Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Lesaint?= Date: Tue, 12 Mar 2019 18:07:24 +0100 Subject: [PATCH] SONAR-11792 update ES JVM options to match defaults in ES 6.6.2 --- .../application/command/EsJvmOptions.java | 81 +++++++++- .../process/ProcessLauncherImpl.java | 36 ++--- .../command/CommandFactoryImplTest.java | 2 +- .../application/command/EsJvmOptionsTest.java | 141 +++++++++++++++--- .../main/java/org/sonar/process/System2.java | 18 +++ 5 files changed, 230 insertions(+), 48 deletions(-) diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/EsJvmOptions.java b/server/sonar-main/src/main/java/org/sonar/application/command/EsJvmOptions.java index 2c3b7bf40ad..0b24606e27a 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/EsJvmOptions.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/EsJvmOptions.java @@ -26,6 +26,7 @@ import java.nio.file.Files; import java.util.LinkedHashMap; import java.util.Map; import java.util.stream.Collectors; +import org.sonar.process.System2; public class EsJvmOptions extends JvmOptions { private static final String ELASTICSEARCH_JVM_OPTIONS_HEADER = "# This file has been automatically generated by SonarQube during startup.\n" + @@ -35,27 +36,95 @@ public class EsJvmOptions extends JvmOptions { "\n"; public EsJvmOptions(File tmpDir) { - super(mandatoryOptions(tmpDir)); + this(System2.INSTANCE, tmpDir); } - private static Map mandatoryOptions(File tmpDir) { + EsJvmOptions(System2 system2, File tmpDir) { + super(mandatoryOptions(system2, tmpDir)); + } + + // this basically writes down the content of jvm.options file distributed in vanilla Elasticsearch package + // with some changes to fit running bundled in SQ + private static Map mandatoryOptions(System2 system2, File tmpDir) { Map res = new LinkedHashMap<>(16); + // GC configuration res.put("-XX:+UseConcMarkSweepGC", ""); res.put("-XX:CMSInitiatingOccupancyFraction=", "75"); res.put("-XX:+UseCMSInitiatingOccupancyOnly", ""); + + // DNS cache policy + // cache ttl in seconds for positive DNS lookups noting that this overrides the + // JDK security property networkaddress.cache.ttl; set to -1 to cache forever + res.put("-Des.networkaddress.cache.ttl=", "60"); + // cache ttl in seconds for negative DNS lookups noting that this overrides the + // JDK security property networkaddress.cache.negative ttl; set to -1 to cache + // forever + res.put("-Des.networkaddress.cache.negative.ttl=", "10"); + + // optimizations + + // pre-touch memory pages used by the JVM during initialization res.put("-XX:+AlwaysPreTouch", ""); - res.put("-Xss", "1m"); + + // basic + // explicitly set the stack size + res.put("-Xss1m", ""); + // set to headless, just in case res.put("-Djava.awt.headless=", "true"); - res.put("-Djava.io.tmpdir=", tmpDir.getAbsolutePath()); + // ensure UTF-8 encoding by default (e.g. filenames) res.put("-Dfile.encoding=", "UTF-8"); + // use our provided JNA always versus the system one res.put("-Djna.nosys=", "true"); - res.put("-Djdk.io.permissionsUseCanonicalPath=", "true"); + + // turn off a JDK optimization that throws away stack traces for common + // exceptions because stack traces are important for debugging + res.put("-XX:-OmitStackTraceInFastThrow", ""); + + // flags to configure Netty res.put("-Dio.netty.noUnsafe=", "true"); res.put("-Dio.netty.noKeySetOptimization=", "true"); res.put("-Dio.netty.recycler.maxCapacityPerThread=", "0"); + + // log4j 2 res.put("-Dlog4j.shutdownHookEnabled=", "false"); res.put("-Dlog4j2.disable.jmx=", "true"); - res.put("-Dlog4j.skipJansi=", "true"); + + // (by default ES 6.6.1 uses variable ${ES_TMPDIR} which is replaced by start scripts. Since we start JAR file + // directly on windows, we specify absolute file as URL (to support space in path) instead + res.put("-Djava.io.tmpdir=", tmpDir.getAbsolutePath()); + + // heap dumps (enable by default in ES 6.6.1, we don't enable them, no one will analyze them anyway) + // generate a heap dump when an allocation from the Java heap fails + // heap dumps are created in the working directory of the JVM + // res.put("-XX:+HeapDumpOnOutOfMemoryError", ""); + // specify an alternative path for heap dumps; ensure the directory exists and + // has sufficient space + // res.put("-XX:HeapDumpPath", "data"); + // specify an alternative path for JVM fatal error logs (ES 6.6.1 default is "logs/hs_err_pid%p.log") + res.put("-XX:ErrorFile=", "../logs/es_hs_err_pid%p.log"); + + // JDK 8 GC logging (by default ES 6.6.1 enables them, we don't want to do that in SQ, no one will analyze them anyway) + // res.put("8:-XX:+PrintGCDetails", ""); + // res.put("8:-XX:+PrintGCDateStamps", ""); + // res.put("8:-XX:+PrintTenuringDistribution", ""); + // res.put("8:-XX:+PrintGCApplicationStoppedTime", ""); + // res.put("8:-Xloggc:logs/gc.log", ""); + // res.put("8:-XX:+UseGCLogFileRotation", ""); + // res.put("8:-XX:NumberOfGCLogFiles", "32"); + // res.put("8:-XX:GCLogFileSize", "64m"); + // JDK 9+ GC logging + // res.put("9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m", ""); + // due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise + // time/date parsing will break in an incompatible way for some date patterns and locals + if (system2.isJava9()) { + res.put("-Djava.locale.providers=", "COMPAT"); + } + + if (system2.isJava10()) { + // temporary workaround for C2 bug with JDK 10 on hardware with AVX-512 + res.put("-XX:UseAVX=", "2"); + } + return res; } diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java index 8b89cd98393..8225f6f5beb 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java @@ -75,10 +75,10 @@ public class ProcessLauncherImpl implements ProcessLauncher { } public ProcessMonitor launch(AbstractCommand command) { - EsInstallation fileSystem = command.getEsInstallation(); - if (fileSystem != null) { - cleanupOutdatedEsData(fileSystem); - writeConfFiles(fileSystem); + EsInstallation esInstallation = command.getEsInstallation(); + if (esInstallation != null) { + cleanupOutdatedEsData(esInstallation); + writeConfFiles(esInstallation); } Process process; @@ -93,7 +93,6 @@ public class ProcessLauncherImpl implements ProcessLauncher { ProcessId processId = command.getProcessId(); try { if (processId == ProcessId.ELASTICSEARCH) { - EsInstallation esInstallation = command.getEsInstallation(); checkArgument(esInstallation != null, "Incorrect configuration EsInstallation is null"); EsConnectorImpl esConnector = new EsConnectorImpl(esInstallation.getClusterName(), singleton(HostAndPort.fromParts(esInstallation.getHost(), esInstallation.getPort()))); return new EsProcessMonitor(process, processId, esConnector); @@ -121,16 +120,17 @@ public class ProcessLauncherImpl implements ProcessLauncher { } private static void cleanupOutdatedEsData(EsInstallation esInstallation) { - esInstallation.getOutdatedSearchDirectories().forEach(outdatedDir -> { - if (outdatedDir.exists()) { - LOG.info("Deleting outdated search index data directory {}", outdatedDir.getAbsolutePath()); - try { - FileUtils2.deleteDirectory(outdatedDir); - } catch (IOException e) { - LOG.info("Failed to delete outdated search index data directory {}", outdatedDir.getAbsolutePath(), e); + esInstallation.getOutdatedSearchDirectories() + .forEach(outdatedDir -> { + if (outdatedDir.exists()) { + LOG.info("Deleting outdated search index data directory {}", outdatedDir.getAbsolutePath()); + try { + FileUtils2.deleteDirectory(outdatedDir); + } catch (IOException e) { + LOG.info("Failed to delete outdated search index data directory {}", outdatedDir.getAbsolutePath(), e); + } } - } - }); + }); } private static void writeConfFiles(EsInstallation esInstallation) { @@ -199,13 +199,13 @@ public class ProcessLauncherImpl implements ProcessLauncher { return create(javaCommand, commands); } - private ProcessBuilder create(AbstractCommand javaCommand, List commands) { + private ProcessBuilder create(AbstractCommand command, List commands) { ProcessBuilder processBuilder = processBuilderSupplier.get(); processBuilder.command(commands); - processBuilder.directory(javaCommand.getWorkDir()); + processBuilder.directory(command.getWorkDir()); Map environment = processBuilder.environment(); - environment.putAll(javaCommand.getEnvVariables()); - javaCommand.getSuppressedEnvVariables().forEach(environment::remove); + environment.putAll(command.getEnvVariables()); + command.getSuppressedEnvVariables().forEach(environment::remove); processBuilder.redirectErrorStream(true); return processBuilder; } diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java index 008556fc3dd..0868f4150a8 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java @@ -215,7 +215,7 @@ public class CommandFactoryImplTest { assertThat(esConfig.getPort()).isEqualTo(1234); assertThat(esConfig.getEsJvmOptions().getAll()) // enforced values - .contains("-XX:+UseConcMarkSweepGC", "-server", "-Dfile.encoding=UTF-8") + .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8") .contains("-Djava.io.tmpdir=" + tempDir.getAbsolutePath()) // user settings .contains("-Xms10G", "-Xmx10G") diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/EsJvmOptionsTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/EsJvmOptionsTest.java index 1ec2e29ba11..a90973c8d34 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/command/EsJvmOptionsTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/command/EsJvmOptionsTest.java @@ -19,16 +19,24 @@ */ package org.sonar.application.command; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.io.File; import java.io.IOException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.sonar.process.System2; import org.sonar.test.ExceptionCauseMatcher; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +@RunWith(DataProviderRunner.class) public class EsJvmOptionsTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -36,29 +44,114 @@ public class EsJvmOptionsTest { public ExpectedException expectedException = ExpectedException.none(); @Test - public void constructor_sets_mandatory_JVM_options() throws IOException { + @UseDataProvider("java8or11") + public void constructor_sets_mandatory_JVM_options_on_Java_8_and_11(System2 system2) throws IOException { File tmpDir = temporaryFolder.newFolder(); - EsJvmOptions underTest = new EsJvmOptions(tmpDir); - - assertThat(underTest.getAll()).containsExactly( - "-XX:+UseConcMarkSweepGC", - "-XX:CMSInitiatingOccupancyFraction=75", - "-XX:+UseCMSInitiatingOccupancyOnly", - "-XX:+AlwaysPreTouch", - "-Xss1m", - "-Djava.awt.headless=true", - "-Djava.io.tmpdir="+ tmpDir.getAbsolutePath(), - "-Dfile.encoding=UTF-8", - "-Djna.nosys=true", - "-Djdk.io.permissionsUseCanonicalPath=true", - "-Dio.netty.noUnsafe=true", - "-Dio.netty.noKeySetOptimization=true", - "-Dio.netty.recycler.maxCapacityPerThread=0", - "-Dlog4j.shutdownHookEnabled=false", - "-Dlog4j2.disable.jmx=true", - "-Dlog4j.skipJansi=true"); + EsJvmOptions underTest = new EsJvmOptions(system2, tmpDir); + + assertThat(underTest.getAll()) + .containsExactly( + "-XX:+UseConcMarkSweepGC", + "-XX:CMSInitiatingOccupancyFraction=75", + "-XX:+UseCMSInitiatingOccupancyOnly", + "-Des.networkaddress.cache.ttl=60", + "-Des.networkaddress.cache.negative.ttl=10", + "-XX:+AlwaysPreTouch", + "-Xss1m", + "-Djava.awt.headless=true", + "-Dfile.encoding=UTF-8", + "-Djna.nosys=true", + "-XX:-OmitStackTraceInFastThrow", + "-Dio.netty.noUnsafe=true", + "-Dio.netty.noKeySetOptimization=true", + "-Dio.netty.recycler.maxCapacityPerThread=0", + "-Dlog4j.shutdownHookEnabled=false", + "-Dlog4j2.disable.jmx=true", + "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath(), + "-XX:ErrorFile=../logs/es_hs_err_pid%p.log"); + } + + @DataProvider + public static Object[][] java8or11() { + System2 java8 = mock(System2.class); + when(java8.isJava9()).thenReturn(false); + when(java8.isJava10()).thenReturn(false); + System2 java10 = mock(System2.class); + when(java10.isJava9()).thenReturn(false); + when(java10.isJava10()).thenReturn(false); + return new Object[][] { + {java8}, + {java10} + }; + } + + @Test + public void constructor_sets_mandatory_JVM_options_on_Java_9() throws IOException { + System2 java9 = mock(System2.class); + when(java9.isJava9()).thenReturn(true); + when(java9.isJava10()).thenReturn(false); + + File tmpDir = temporaryFolder.newFolder(); + EsJvmOptions underTest = new EsJvmOptions(java9, tmpDir); + + assertThat(underTest.getAll()) + .containsExactly( + "-XX:+UseConcMarkSweepGC", + "-XX:CMSInitiatingOccupancyFraction=75", + "-XX:+UseCMSInitiatingOccupancyOnly", + "-Des.networkaddress.cache.ttl=60", + "-Des.networkaddress.cache.negative.ttl=10", + "-XX:+AlwaysPreTouch", + "-Xss1m", + "-Djava.awt.headless=true", + "-Dfile.encoding=UTF-8", + "-Djna.nosys=true", + "-XX:-OmitStackTraceInFastThrow", + "-Dio.netty.noUnsafe=true", + "-Dio.netty.noKeySetOptimization=true", + "-Dio.netty.recycler.maxCapacityPerThread=0", + "-Dlog4j.shutdownHookEnabled=false", + "-Dlog4j2.disable.jmx=true", + "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath(), + "-XX:ErrorFile=../logs/es_hs_err_pid%p.log", + "-Djava.locale.providers=COMPAT"); } + @Test + public void constructor_sets_mandatory_JVM_options_on_Java_10() throws IOException { + System2 java10 = mock(System2.class); + when(java10.isJava9()).thenReturn(false); + when(java10.isJava10()).thenReturn(true); + + File tmpDir = temporaryFolder.newFolder(); + EsJvmOptions underTest = new EsJvmOptions(java10, tmpDir); + + assertThat(underTest.getAll()) + .containsExactly( + "-XX:+UseConcMarkSweepGC", + "-XX:CMSInitiatingOccupancyFraction=75", + "-XX:+UseCMSInitiatingOccupancyOnly", + "-Des.networkaddress.cache.ttl=60", + "-Des.networkaddress.cache.negative.ttl=10", + "-XX:+AlwaysPreTouch", + "-Xss1m", + "-Djava.awt.headless=true", + "-Dfile.encoding=UTF-8", + "-Djna.nosys=true", + "-XX:-OmitStackTraceInFastThrow", + "-Dio.netty.noUnsafe=true", + "-Dio.netty.noKeySetOptimization=true", + "-Dio.netty.recycler.maxCapacityPerThread=0", + "-Dlog4j.shutdownHookEnabled=false", + "-Dlog4j2.disable.jmx=true", + "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath(), + "-XX:ErrorFile=../logs/es_hs_err_pid%p.log", + "-XX:UseAVX=2"); + } + + /** + * This test may fail if SQ's test are not executed with target Java version 8. + */ @Test public void writeToJvmOptionFile_writes_all_JVM_options_to_file_with_warning_header() throws IOException { File tmpDir = temporaryFolder.newFolder("with space"); @@ -78,19 +171,21 @@ public class EsJvmOptionsTest { "-XX:+UseConcMarkSweepGC\n" + "-XX:CMSInitiatingOccupancyFraction=75\n" + "-XX:+UseCMSInitiatingOccupancyOnly\n" + + "-Des.networkaddress.cache.ttl=60\n" + + "-Des.networkaddress.cache.negative.ttl=10\n" + "-XX:+AlwaysPreTouch\n" + "-Xss1m\n" + "-Djava.awt.headless=true\n" + - "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath() + "\n" + "-Dfile.encoding=UTF-8\n" + "-Djna.nosys=true\n" + - "-Djdk.io.permissionsUseCanonicalPath=true\n" + + "-XX:-OmitStackTraceInFastThrow\n" + "-Dio.netty.noUnsafe=true\n" + "-Dio.netty.noKeySetOptimization=true\n" + "-Dio.netty.recycler.maxCapacityPerThread=0\n" + "-Dlog4j.shutdownHookEnabled=false\n" + "-Dlog4j2.disable.jmx=true\n" + - "-Dlog4j.skipJansi=true\n" + + "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath() + "\n" + + "-XX:ErrorFile=../logs/es_hs_err_pid%p.log\n" + "-foo\n" + "-bar"); diff --git a/server/sonar-process/src/main/java/org/sonar/process/System2.java b/server/sonar-process/src/main/java/org/sonar/process/System2.java index 93a46cc3dd0..e74ba7d4dea 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/System2.java +++ b/server/sonar-process/src/main/java/org/sonar/process/System2.java @@ -41,6 +41,14 @@ public interface System2 { public boolean isOsWindows() { return SystemUtils.IS_OS_WINDOWS; } + + public boolean isJava9() { + return SystemUtils.JAVA_VERSION != null && SystemUtils.JAVA_VERSION.startsWith("9"); + } + + public boolean isJava10() { + return SystemUtils.JAVA_VERSION != null && SystemUtils.JAVA_VERSION.startsWith("10"); + } }; /** @@ -57,4 +65,14 @@ public interface System2 { * True if this is MS Windows. */ boolean isOsWindows(); + + /** + * True is current Java version is Java 9. + */ + boolean isJava9(); + + /** + * True is current Java version is Java 10. + */ + boolean isJava10(); } -- 2.39.5