import java.util.LinkedHashMap;
import java.util.Map;
import java.util.stream.Collectors;
+import org.sonar.process.System2;
public class EsJvmOptions extends JvmOptions<EsJvmOptions> {
private static final String ELASTICSEARCH_JVM_OPTIONS_HEADER = "# This file has been automatically generated by SonarQube during startup.\n" +
"\n";
public EsJvmOptions(File tmpDir) {
- super(mandatoryOptions(tmpDir));
+ this(System2.INSTANCE, tmpDir);
}
- private static Map<String, String> mandatoryOptions(File tmpDir) {
+ EsJvmOptions(System2 system2, File tmpDir) {
+ super(mandatoryOptions(system2, tmpDir));
+ }
+
+ // this basically writes down the content of jvm.options file distributed in vanilla Elasticsearch package
+ // with some changes to fit running bundled in SQ
+ private static Map<String, String> mandatoryOptions(System2 system2, File tmpDir) {
Map<String, String> res = new LinkedHashMap<>(16);
+ // GC configuration
res.put("-XX:+UseConcMarkSweepGC", "");
res.put("-XX:CMSInitiatingOccupancyFraction=", "75");
res.put("-XX:+UseCMSInitiatingOccupancyOnly", "");
+
+ // DNS cache policy
+ // cache ttl in seconds for positive DNS lookups noting that this overrides the
+ // JDK security property networkaddress.cache.ttl; set to -1 to cache forever
+ res.put("-Des.networkaddress.cache.ttl=", "60");
+ // cache ttl in seconds for negative DNS lookups noting that this overrides the
+ // JDK security property networkaddress.cache.negative ttl; set to -1 to cache
+ // forever
+ res.put("-Des.networkaddress.cache.negative.ttl=", "10");
+
+ // optimizations
+
+ // pre-touch memory pages used by the JVM during initialization
res.put("-XX:+AlwaysPreTouch", "");
- res.put("-Xss", "1m");
+
+ // basic
+ // explicitly set the stack size
+ res.put("-Xss1m", "");
+ // set to headless, just in case
res.put("-Djava.awt.headless=", "true");
- res.put("-Djava.io.tmpdir=", tmpDir.getAbsolutePath());
+ // ensure UTF-8 encoding by default (e.g. filenames)
res.put("-Dfile.encoding=", "UTF-8");
+ // use our provided JNA always versus the system one
res.put("-Djna.nosys=", "true");
- res.put("-Djdk.io.permissionsUseCanonicalPath=", "true");
+
+ // turn off a JDK optimization that throws away stack traces for common
+ // exceptions because stack traces are important for debugging
+ res.put("-XX:-OmitStackTraceInFastThrow", "");
+
+ // flags to configure Netty
res.put("-Dio.netty.noUnsafe=", "true");
res.put("-Dio.netty.noKeySetOptimization=", "true");
res.put("-Dio.netty.recycler.maxCapacityPerThread=", "0");
+
+ // log4j 2
res.put("-Dlog4j.shutdownHookEnabled=", "false");
res.put("-Dlog4j2.disable.jmx=", "true");
- res.put("-Dlog4j.skipJansi=", "true");
+
+ // (by default ES 6.6.1 uses variable ${ES_TMPDIR} which is replaced by start scripts. Since we start JAR file
+ // directly on windows, we specify absolute file as URL (to support space in path) instead
+ res.put("-Djava.io.tmpdir=", tmpDir.getAbsolutePath());
+
+ // heap dumps (enable by default in ES 6.6.1, we don't enable them, no one will analyze them anyway)
+ // generate a heap dump when an allocation from the Java heap fails
+ // heap dumps are created in the working directory of the JVM
+ // res.put("-XX:+HeapDumpOnOutOfMemoryError", "");
+ // specify an alternative path for heap dumps; ensure the directory exists and
+ // has sufficient space
+ // res.put("-XX:HeapDumpPath", "data");
+ // specify an alternative path for JVM fatal error logs (ES 6.6.1 default is "logs/hs_err_pid%p.log")
+ res.put("-XX:ErrorFile=", "../logs/es_hs_err_pid%p.log");
+
+ // JDK 8 GC logging (by default ES 6.6.1 enables them, we don't want to do that in SQ, no one will analyze them anyway)
+ // res.put("8:-XX:+PrintGCDetails", "");
+ // res.put("8:-XX:+PrintGCDateStamps", "");
+ // res.put("8:-XX:+PrintTenuringDistribution", "");
+ // res.put("8:-XX:+PrintGCApplicationStoppedTime", "");
+ // res.put("8:-Xloggc:logs/gc.log", "");
+ // res.put("8:-XX:+UseGCLogFileRotation", "");
+ // res.put("8:-XX:NumberOfGCLogFiles", "32");
+ // res.put("8:-XX:GCLogFileSize", "64m");
+ // JDK 9+ GC logging
+ // res.put("9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m", "");
+ // due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
+ // time/date parsing will break in an incompatible way for some date patterns and locals
+ if (system2.isJava9()) {
+ res.put("-Djava.locale.providers=", "COMPAT");
+ }
+
+ if (system2.isJava10()) {
+ // temporary workaround for C2 bug with JDK 10 on hardware with AVX-512
+ res.put("-XX:UseAVX=", "2");
+ }
+
return res;
}
}
public ProcessMonitor launch(AbstractCommand command) {
- EsInstallation fileSystem = command.getEsInstallation();
- if (fileSystem != null) {
- cleanupOutdatedEsData(fileSystem);
- writeConfFiles(fileSystem);
+ EsInstallation esInstallation = command.getEsInstallation();
+ if (esInstallation != null) {
+ cleanupOutdatedEsData(esInstallation);
+ writeConfFiles(esInstallation);
}
Process process;
ProcessId processId = command.getProcessId();
try {
if (processId == ProcessId.ELASTICSEARCH) {
- EsInstallation esInstallation = command.getEsInstallation();
checkArgument(esInstallation != null, "Incorrect configuration EsInstallation is null");
EsConnectorImpl esConnector = new EsConnectorImpl(esInstallation.getClusterName(), singleton(HostAndPort.fromParts(esInstallation.getHost(), esInstallation.getPort())));
return new EsProcessMonitor(process, processId, esConnector);
}
private static void cleanupOutdatedEsData(EsInstallation esInstallation) {
- esInstallation.getOutdatedSearchDirectories().forEach(outdatedDir -> {
- if (outdatedDir.exists()) {
- LOG.info("Deleting outdated search index data directory {}", outdatedDir.getAbsolutePath());
- try {
- FileUtils2.deleteDirectory(outdatedDir);
- } catch (IOException e) {
- LOG.info("Failed to delete outdated search index data directory {}", outdatedDir.getAbsolutePath(), e);
+ esInstallation.getOutdatedSearchDirectories()
+ .forEach(outdatedDir -> {
+ if (outdatedDir.exists()) {
+ LOG.info("Deleting outdated search index data directory {}", outdatedDir.getAbsolutePath());
+ try {
+ FileUtils2.deleteDirectory(outdatedDir);
+ } catch (IOException e) {
+ LOG.info("Failed to delete outdated search index data directory {}", outdatedDir.getAbsolutePath(), e);
+ }
}
- }
- });
+ });
}
private static void writeConfFiles(EsInstallation esInstallation) {
return create(javaCommand, commands);
}
- private ProcessBuilder create(AbstractCommand<?> javaCommand, List<String> commands) {
+ private ProcessBuilder create(AbstractCommand<?> command, List<String> commands) {
ProcessBuilder processBuilder = processBuilderSupplier.get();
processBuilder.command(commands);
- processBuilder.directory(javaCommand.getWorkDir());
+ processBuilder.directory(command.getWorkDir());
Map<String, String> environment = processBuilder.environment();
- environment.putAll(javaCommand.getEnvVariables());
- javaCommand.getSuppressedEnvVariables().forEach(environment::remove);
+ environment.putAll(command.getEnvVariables());
+ command.getSuppressedEnvVariables().forEach(environment::remove);
processBuilder.redirectErrorStream(true);
return processBuilder;
}
assertThat(esConfig.getPort()).isEqualTo(1234);
assertThat(esConfig.getEsJvmOptions().getAll())
// enforced values
- .contains("-XX:+UseConcMarkSweepGC", "-server", "-Dfile.encoding=UTF-8")
+ .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8")
.contains("-Djava.io.tmpdir=" + tempDir.getAbsolutePath())
// user settings
.contains("-Xms10G", "-Xmx10G")
*/
package org.sonar.application.command;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
import java.io.File;
import java.io.IOException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.sonar.process.System2;
import org.sonar.test.ExceptionCauseMatcher;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+@RunWith(DataProviderRunner.class)
public class EsJvmOptionsTest {
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
public ExpectedException expectedException = ExpectedException.none();
@Test
- public void constructor_sets_mandatory_JVM_options() throws IOException {
+ @UseDataProvider("java8or11")
+ public void constructor_sets_mandatory_JVM_options_on_Java_8_and_11(System2 system2) throws IOException {
File tmpDir = temporaryFolder.newFolder();
- EsJvmOptions underTest = new EsJvmOptions(tmpDir);
-
- assertThat(underTest.getAll()).containsExactly(
- "-XX:+UseConcMarkSweepGC",
- "-XX:CMSInitiatingOccupancyFraction=75",
- "-XX:+UseCMSInitiatingOccupancyOnly",
- "-XX:+AlwaysPreTouch",
- "-Xss1m",
- "-Djava.awt.headless=true",
- "-Djava.io.tmpdir="+ tmpDir.getAbsolutePath(),
- "-Dfile.encoding=UTF-8",
- "-Djna.nosys=true",
- "-Djdk.io.permissionsUseCanonicalPath=true",
- "-Dio.netty.noUnsafe=true",
- "-Dio.netty.noKeySetOptimization=true",
- "-Dio.netty.recycler.maxCapacityPerThread=0",
- "-Dlog4j.shutdownHookEnabled=false",
- "-Dlog4j2.disable.jmx=true",
- "-Dlog4j.skipJansi=true");
+ EsJvmOptions underTest = new EsJvmOptions(system2, tmpDir);
+
+ assertThat(underTest.getAll())
+ .containsExactly(
+ "-XX:+UseConcMarkSweepGC",
+ "-XX:CMSInitiatingOccupancyFraction=75",
+ "-XX:+UseCMSInitiatingOccupancyOnly",
+ "-Des.networkaddress.cache.ttl=60",
+ "-Des.networkaddress.cache.negative.ttl=10",
+ "-XX:+AlwaysPreTouch",
+ "-Xss1m",
+ "-Djava.awt.headless=true",
+ "-Dfile.encoding=UTF-8",
+ "-Djna.nosys=true",
+ "-XX:-OmitStackTraceInFastThrow",
+ "-Dio.netty.noUnsafe=true",
+ "-Dio.netty.noKeySetOptimization=true",
+ "-Dio.netty.recycler.maxCapacityPerThread=0",
+ "-Dlog4j.shutdownHookEnabled=false",
+ "-Dlog4j2.disable.jmx=true",
+ "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath(),
+ "-XX:ErrorFile=../logs/es_hs_err_pid%p.log");
+ }
+
+ @DataProvider
+ public static Object[][] java8or11() {
+ System2 java8 = mock(System2.class);
+ when(java8.isJava9()).thenReturn(false);
+ when(java8.isJava10()).thenReturn(false);
+ System2 java10 = mock(System2.class);
+ when(java10.isJava9()).thenReturn(false);
+ when(java10.isJava10()).thenReturn(false);
+ return new Object[][] {
+ {java8},
+ {java10}
+ };
+ }
+
+ @Test
+ public void constructor_sets_mandatory_JVM_options_on_Java_9() throws IOException {
+ System2 java9 = mock(System2.class);
+ when(java9.isJava9()).thenReturn(true);
+ when(java9.isJava10()).thenReturn(false);
+
+ File tmpDir = temporaryFolder.newFolder();
+ EsJvmOptions underTest = new EsJvmOptions(java9, tmpDir);
+
+ assertThat(underTest.getAll())
+ .containsExactly(
+ "-XX:+UseConcMarkSweepGC",
+ "-XX:CMSInitiatingOccupancyFraction=75",
+ "-XX:+UseCMSInitiatingOccupancyOnly",
+ "-Des.networkaddress.cache.ttl=60",
+ "-Des.networkaddress.cache.negative.ttl=10",
+ "-XX:+AlwaysPreTouch",
+ "-Xss1m",
+ "-Djava.awt.headless=true",
+ "-Dfile.encoding=UTF-8",
+ "-Djna.nosys=true",
+ "-XX:-OmitStackTraceInFastThrow",
+ "-Dio.netty.noUnsafe=true",
+ "-Dio.netty.noKeySetOptimization=true",
+ "-Dio.netty.recycler.maxCapacityPerThread=0",
+ "-Dlog4j.shutdownHookEnabled=false",
+ "-Dlog4j2.disable.jmx=true",
+ "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath(),
+ "-XX:ErrorFile=../logs/es_hs_err_pid%p.log",
+ "-Djava.locale.providers=COMPAT");
}
+ @Test
+ public void constructor_sets_mandatory_JVM_options_on_Java_10() throws IOException {
+ System2 java10 = mock(System2.class);
+ when(java10.isJava9()).thenReturn(false);
+ when(java10.isJava10()).thenReturn(true);
+
+ File tmpDir = temporaryFolder.newFolder();
+ EsJvmOptions underTest = new EsJvmOptions(java10, tmpDir);
+
+ assertThat(underTest.getAll())
+ .containsExactly(
+ "-XX:+UseConcMarkSweepGC",
+ "-XX:CMSInitiatingOccupancyFraction=75",
+ "-XX:+UseCMSInitiatingOccupancyOnly",
+ "-Des.networkaddress.cache.ttl=60",
+ "-Des.networkaddress.cache.negative.ttl=10",
+ "-XX:+AlwaysPreTouch",
+ "-Xss1m",
+ "-Djava.awt.headless=true",
+ "-Dfile.encoding=UTF-8",
+ "-Djna.nosys=true",
+ "-XX:-OmitStackTraceInFastThrow",
+ "-Dio.netty.noUnsafe=true",
+ "-Dio.netty.noKeySetOptimization=true",
+ "-Dio.netty.recycler.maxCapacityPerThread=0",
+ "-Dlog4j.shutdownHookEnabled=false",
+ "-Dlog4j2.disable.jmx=true",
+ "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath(),
+ "-XX:ErrorFile=../logs/es_hs_err_pid%p.log",
+ "-XX:UseAVX=2");
+ }
+
+ /**
+ * This test may fail if SQ's test are not executed with target Java version 8.
+ */
@Test
public void writeToJvmOptionFile_writes_all_JVM_options_to_file_with_warning_header() throws IOException {
File tmpDir = temporaryFolder.newFolder("with space");
"-XX:+UseConcMarkSweepGC\n" +
"-XX:CMSInitiatingOccupancyFraction=75\n" +
"-XX:+UseCMSInitiatingOccupancyOnly\n" +
+ "-Des.networkaddress.cache.ttl=60\n" +
+ "-Des.networkaddress.cache.negative.ttl=10\n" +
"-XX:+AlwaysPreTouch\n" +
"-Xss1m\n" +
"-Djava.awt.headless=true\n" +
- "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath() + "\n" +
"-Dfile.encoding=UTF-8\n" +
"-Djna.nosys=true\n" +
- "-Djdk.io.permissionsUseCanonicalPath=true\n" +
+ "-XX:-OmitStackTraceInFastThrow\n" +
"-Dio.netty.noUnsafe=true\n" +
"-Dio.netty.noKeySetOptimization=true\n" +
"-Dio.netty.recycler.maxCapacityPerThread=0\n" +
"-Dlog4j.shutdownHookEnabled=false\n" +
"-Dlog4j2.disable.jmx=true\n" +
- "-Dlog4j.skipJansi=true\n" +
+ "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath() + "\n" +
+ "-XX:ErrorFile=../logs/es_hs_err_pid%p.log\n" +
"-foo\n" +
"-bar");
public boolean isOsWindows() {
return SystemUtils.IS_OS_WINDOWS;
}
+
+ public boolean isJava9() {
+ return SystemUtils.JAVA_VERSION != null && SystemUtils.JAVA_VERSION.startsWith("9");
+ }
+
+ public boolean isJava10() {
+ return SystemUtils.JAVA_VERSION != null && SystemUtils.JAVA_VERSION.startsWith("10");
+ }
};
/**
* True if this is MS Windows.
*/
boolean isOsWindows();
+
+ /**
+ * True is current Java version is Java 9.
+ */
+ boolean isJava9();
+
+ /**
+ * True is current Java version is Java 10.
+ */
+ boolean isJava10();
}
projects / sonarqube.git / commitdiff