From fba12617210b4fda3f97fed9491f111588443d70 Mon Sep 17 00:00:00 2001 From: Jacek Poreda Date: Mon, 6 Feb 2023 09:10:02 +0100 Subject: [PATCH] SONAR-17714 Update ES binaries - use no-jdk for local development and jdk for open source part - use ES cli tool to start Elasticsearch - platform agnostic - fix query issue with dates - disable security autoconfiguration on a single node environment - fix single-node settings - Use Elasticsearch CLI java tool to run keystore cli and Elasticsearch --- gradle.properties | 6 +- .../application/ProcessLauncherImpl.java | 30 +----- .../application/command/CommandFactory.java | 6 +- .../command/CommandFactoryImpl.java | 50 ++-------- .../application/command/EsJvmOptions.java | 4 +- ...ommand.java => EsServerCliJvmOptions.java} | 36 +++---- .../sonar/application/es/EsInstallation.java | 4 +- .../sonar/application/es/EsKeyStoreCli.java | 17 ++-- .../org/sonar/application/es/EsSettings.java | 20 ++-- .../application/ProcessLauncherImplTest.java | 10 +- .../sonar/application/SchedulerImplTest.java | 15 ++- .../command/CommandFactoryImplTest.java | 62 +----------- .../application/command/EsJvmOptionsTest.java | 4 +- .../application/es/EsInstallationTest.java | 4 +- .../application/es/EsKeyStoreCliTest.java | 15 +-- .../sonar/application/es/EsSettingsTest.java | 5 +- .../java/org/sonar/server/es/BaseDoc.java | 7 -- .../java/org/sonar/server/es/EsClient.java | 4 +- .../sonar/server/es/newindex/FieldAware.java | 2 +- .../rule/index/RuleIndexDefinition.java | 3 +- .../sonar/server/issue/index/IssueIndex.java | 12 +-- sonar-application/build.gradle | 96 ++++++++++--------- .../elasticsearch-patch/bin/elasticsearch | 81 ---------------- 23 files changed, 155 insertions(+), 338 deletions(-) rename server/sonar-main/src/main/java/org/sonar/application/command/{EsScriptCommand.java => EsServerCliJvmOptions.java} (50%) delete mode 100755 sonar-application/src/main/assembly/elasticsearch-patch/bin/elasticsearch diff --git a/gradle.properties b/gradle.properties index 57533140a19..fdb8533a051 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,6 +9,8 @@ org.gradle.vfs.watch=true # https://www.elastic.co/downloads/elasticsearch-no-jdk elasticsearchDownloadUrlPath=https://artifacts.elastic.co/downloads/elasticsearch/ elasticsearchDownloadRepoxUrlPath=https://repox.jfrog.io/artifactory/sonarsource-bucket/sonarqube/elasticsearch/ -elasticsearchDownloadUrlFile=elasticsearch-7.17.8-no-jdk-linux-x86_64.tar.gz -elasticsearchDownloadSha512=904e049ba42335fe979f055b586d616f04e269caa16ffe285e8e30df31971aa9739b334d7471e19afd477103b541e8400a91aeecc4df7825d246c361ccdad29b +elasticsearchDownloadUrlFileNoJdk=elasticsearch-8.6.1-no-jdk-linux-x86_64.tar.gz +elasticsearchDownloadUrlFileJdk=elasticsearch-8.6.1-linux-x86_64.tar.gz +elasticsearchDownloadSha512NoJdk=b0da32bcd3edd78214164530c3fdeee6e7e0cd5e8a965251ebf1423bdd809a411fa654d4dd6dae37e5bf84174b4f5988f7388ce86d9363150d78ca87805d6d8a +elasticsearchDownloadSha512Jdk=29ba49201bd4dd0c8b2e7a1cd96d90cc2876b40ee16b1cbc53f01bc450652b3d540522e6c92fb32e8cd972a67370dbdb4c4ae1e0c9fd570536b3da62cc8da65e projectType=application diff --git a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java index eb9d72d8ba9..f8c2952070e 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java @@ -37,7 +37,6 @@ import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.application.command.AbstractCommand; -import org.sonar.application.command.EsScriptCommand; import org.sonar.application.command.JavaCommand; import org.sonar.application.command.JvmOptions; import org.sonar.application.es.EsConnectorImpl; @@ -54,7 +53,6 @@ import org.sonar.process.sharedmemoryfile.ProcessCommands; import static com.google.common.base.Preconditions.checkArgument; import static java.lang.String.format; import static java.util.Collections.singleton; -import static java.util.Objects.requireNonNull; import static org.sonar.application.es.EsKeyStoreCli.BOOTSTRAP_PASSWORD_PROPERTY_KEY; import static org.sonar.application.es.EsKeyStoreCli.KEYSTORE_PASSWORD_PROPERTY_KEY; import static org.sonar.application.es.EsKeyStoreCli.TRUSTSTORE_PASSWORD_PROPERTY_KEY; @@ -93,10 +91,8 @@ public class ProcessLauncherImpl implements ProcessLauncher { } Process process; - if (command instanceof EsScriptCommand esScriptCommand) { - process = launchExternal(esScriptCommand); - } else if (command instanceof JavaCommand) { - process = launchJava((JavaCommand) command); + if (command instanceof JavaCommand javaCommand) { + process = launchJava(javaCommand); } else { throw new IllegalStateException("Unexpected type of command: " + command.getClass()); } @@ -121,16 +117,6 @@ public class ProcessLauncherImpl implements ProcessLauncher { } } - private Process launchExternal(EsScriptCommand esScriptCommand) { - try { - ProcessBuilder processBuilder = create(esScriptCommand); - logLaunchedCommand(esScriptCommand, processBuilder); - return processBuilder.start(); - } catch (Exception e) { - throw new IllegalStateException(format("Fail to launch process [%s]", esScriptCommand.getProcessId().getHumanReadableName()), e); - } - } - private static void cleanupOutdatedEsData(EsInstallation esInstallation) { esInstallation.getOutdatedSearchDirectories() .forEach(outdatedDir -> { @@ -204,7 +190,7 @@ public class ProcessLauncherImpl implements ProcessLauncher { try { Files.copy(from, to, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { - throw new IllegalStateException("Could not copy file: " + from.toString(), e); + throw new IllegalStateException("Could not copy file: " + from, e); } } @@ -236,16 +222,6 @@ public class ProcessLauncherImpl implements ProcessLauncher { } } - private ProcessBuilder create(EsScriptCommand esScriptCommand) { - List commands = new ArrayList<>(); - EsInstallation esInstallation = esScriptCommand.getEsInstallation(); - requireNonNull(esInstallation, () -> "No Elasticsearch installation configuration is available for the command of type " + esScriptCommand.getClass()); - commands.add(esInstallation.getExecutable().getAbsolutePath()); - commands.addAll(esScriptCommand.getOptions()); - - return create(esScriptCommand, commands); - } - private ProcessBuilder create(JavaCommand javaCommand) { List commands = new ArrayList<>(); commands.add(buildJavaPath()); diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactory.java b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactory.java index 0ab0b3aaa32..a0c32f7382c 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactory.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactory.java @@ -21,10 +21,10 @@ package org.sonar.application.command; public interface CommandFactory { - AbstractCommand createEsCommand(); + JavaCommand createEsCommand(); - JavaCommand createWebCommand(boolean leader); + JavaCommand createWebCommand(boolean leader); - JavaCommand createCeCommand(); + JavaCommand createCeCommand(); } diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java index 24f52c64451..9e288739331 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java @@ -20,6 +20,7 @@ package org.sonar.application.command; import java.io.File; +import java.nio.file.Paths; import java.util.Map; import java.util.Optional; import org.slf4j.LoggerFactory; @@ -74,12 +75,10 @@ public class CommandFactoryImpl implements CommandFactory { private static final Version SQ_VERSION = MetadataLoader.loadSQVersion(org.sonar.api.utils.System2.INSTANCE); private final Props props; private final File tempDir; - private final System2 system2; public CommandFactoryImpl(Props props, File tempDir, System2 system2) { this.props = props; this.tempDir = tempDir; - this.system2 = system2; String javaToolOptions = system2.getenv(ENV_VAR_JAVA_TOOL_OPTIONS); if (javaToolOptions != null && !javaToolOptions.trim().isEmpty()) { LoggerFactory.getLogger(CommandFactoryImpl.class) @@ -92,52 +91,23 @@ public class CommandFactoryImpl implements CommandFactory { .warn("ES_JAVA_OPTS is defined but will be ignored. " + "Use properties sonar.search.javaOpts and/or sonar.search.javaAdditionalOpts in sonar.properties to change SQ JVM processes options"); } - } @Override - public AbstractCommand createEsCommand() { - if (system2.isOsWindows()) { - return createEsCommandForWindows(); - } - return createEsCommandForUnix(); - } - - private EsScriptCommand createEsCommandForUnix() { - EsInstallation esInstallation = createEsInstallation(); - return new EsScriptCommand(ProcessId.ELASTICSEARCH, esInstallation.getHomeDirectory()) - .setEsInstallation(esInstallation) - .setEnvVariable("ES_PATH_CONF", esInstallation.getConfDirectory().getAbsolutePath()) - .setEnvVariable("ES_JVM_OPTIONS", esInstallation.getJvmOptions().getAbsolutePath()) - .setEnvVariable("ES_JAVA_HOME", System.getProperties().getProperty("java.home")) - .setEnvVariable("LIBFFI_TMPDIR", this.tempDir.getAbsolutePath()) - .suppressEnvVariable(ENV_VAR_JAVA_TOOL_OPTIONS) - .suppressEnvVariable(ENV_VAR_ES_JAVA_OPTS); - } - - private JavaCommand createEsCommandForWindows() { + public JavaCommand createEsCommand() { EsInstallation esInstallation = createEsInstallation(); - return new JavaCommand(ProcessId.ELASTICSEARCH, esInstallation.getHomeDirectory()) + String esHomeDirectoryAbsolutePath = esInstallation.getHomeDirectory().getAbsolutePath(); + return new JavaCommand(ProcessId.ELASTICSEARCH, esInstallation.getHomeDirectory()) .setEsInstallation(esInstallation) - .setReadsArgumentsFromFile(false) - .setJvmOptions(esInstallation.getEsJvmOptions() - .add("-Delasticsearch") - .add("-Des.path.home=" + esInstallation.getHomeDirectory().getAbsolutePath()) - .add("-Des.path.conf=" + esInstallation.getConfDirectory().getAbsolutePath())) - .setEnvVariable("ES_JVM_OPTIONS", esInstallation.getJvmOptions().getAbsolutePath()) + .setJvmOptions(new EsServerCliJvmOptions(esInstallation)) .setEnvVariable("ES_JAVA_HOME", System.getProperties().getProperty("java.home")) - .setEnvVariable("LIBFFI_TMPDIR", this.tempDir.getAbsolutePath()) - .setClassName("org.elasticsearch.bootstrap.Elasticsearch") - .addClasspath("lib/*") - .suppressEnvVariable(ENV_VAR_JAVA_TOOL_OPTIONS) - .suppressEnvVariable(ENV_VAR_ES_JAVA_OPTS); + .setClassName("org.elasticsearch.launcher.CliToolLauncher") + .addClasspath(Paths.get(esHomeDirectoryAbsolutePath, "lib").toAbsolutePath() + File.separator + "*") + .addClasspath(Paths.get(esHomeDirectoryAbsolutePath, "lib", "cli-launcher").toAbsolutePath() + File.separator + "*"); } private EsInstallation createEsInstallation() { EsInstallation esInstallation = new EsInstallation(props); - if (!esInstallation.getExecutable().exists()) { - throw new IllegalStateException("Cannot find elasticsearch binary"); - } Map settingsMap = new EsSettings(props, esInstallation, System2.INSTANCE).build(); esInstallation @@ -152,7 +122,7 @@ public class CommandFactoryImpl implements CommandFactory { } @Override - public JavaCommand createWebCommand(boolean leader) { + public JavaCommand createWebCommand(boolean leader) { File homeDir = props.nonNullValueAsFile(PATH_HOME.getKey()); WebJvmOptions jvmOptions = new WebJvmOptions(tempDir) @@ -179,7 +149,7 @@ public class CommandFactoryImpl implements CommandFactory { } @Override - public JavaCommand createCeCommand() { + public JavaCommand createCeCommand() { File homeDir = props.nonNullValueAsFile(PATH_HOME.getKey()); CeJvmOptions jvmOptions = new CeJvmOptions(tempDir) diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/EsJvmOptions.java b/server/sonar-main/src/main/java/org/sonar/application/command/EsJvmOptions.java index 64b00823c73..0d392da5fc0 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/EsJvmOptions.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/EsJvmOptions.java @@ -74,8 +74,8 @@ public class EsJvmOptions extends JvmOptions { // res.put("-XX:HeapDumpPath", "data"); // specify an alternative path for JVM fatal error logs (ES 6.6.1 default is "logs/hs_err_pid%p.log") var path = Paths.get(props.value("sonar.path.logs", "logs"), "es_hs_err_pid%p.log"); - res.put("-XX:ErrorFile=", path.toAbsolutePath().toString()); - + res.put("-XX:ErrorFile=", path.toAbsolutePath().toString()); + res.put("-Xlog:disable", ""); } /** diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/EsScriptCommand.java b/server/sonar-main/src/main/java/org/sonar/application/command/EsServerCliJvmOptions.java similarity index 50% rename from server/sonar-main/src/main/java/org/sonar/application/command/EsScriptCommand.java rename to server/sonar-main/src/main/java/org/sonar/application/command/EsServerCliJvmOptions.java index 0d3fa10563f..f5f7030dc20 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/EsScriptCommand.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/EsServerCliJvmOptions.java @@ -19,27 +19,27 @@ */ package org.sonar.application.command; -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import org.sonar.process.ProcessId; -import org.sonar.process.System2; +import java.util.LinkedHashMap; +import java.util.Map; +import org.sonar.application.es.EsInstallation; -public class EsScriptCommand extends AbstractCommand { - private List options = new ArrayList<>(); +class EsServerCliJvmOptions extends JvmOptions { - public EsScriptCommand(ProcessId id, File workDir) { - super(id, workDir, System2.INSTANCE); + public EsServerCliJvmOptions(EsInstallation esInstallation) { + super(mandatoryOptions(esInstallation)); } - public List getOptions() { - return options; - } - - public EsScriptCommand addOption(String s) { - if (!s.isEmpty()) { - options.add(s); - } - return this; + private static Map mandatoryOptions(EsInstallation esInstallation) { + Map res = new LinkedHashMap<>(9); + res.put("-Xms4m", ""); + res.put("-Xmx64m", ""); + res.put("-XX:+UseSerialGC", ""); + res.put("-Dcli.name=", "server"); + res.put("-Dcli.script=", "./bin/elasticsearch"); + res.put("-Dcli.libs=", "lib/tools/server-cli"); + res.put("-Des.path.home=", esInstallation.getHomeDirectory().getAbsolutePath()); + res.put("-Des.path.conf=", esInstallation.getConfDirectory().getAbsolutePath()); + res.put("-Des.distribution.type=", "tar"); + return res; } } diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java index 3c44de3d16c..ebda731f753 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java @@ -97,14 +97,14 @@ public class EsInstallation { private static List buildOutdatedSearchDirs(Props props) { String dataPath = props.nonNullValue(PATH_DATA.getKey()); - return Stream.of("es", "es5", "es6") + return Stream.of("es", "es5", "es6", "es7") .map(t -> new File(dataPath, t)) .collect(MoreCollectors.toList()); } private static File buildDataDir(Props props) { String dataPath = props.nonNullValue(PATH_DATA.getKey()); - return new File(dataPath, "es7"); + return new File(dataPath, "es8"); } private static File buildLogPath(Props props) { diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsKeyStoreCli.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsKeyStoreCli.java index ba7fceab342..74514120567 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsKeyStoreCli.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsKeyStoreCli.java @@ -20,6 +20,7 @@ package org.sonar.application.es; import java.io.BufferedWriter; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; @@ -41,19 +42,18 @@ public class EsKeyStoreCli { public static final String KEYSTORE_PASSWORD_PROPERTY_KEY = "xpack.security.transport.ssl.keystore.secure_password"; public static final String TRUSTSTORE_PASSWORD_PROPERTY_KEY = "xpack.security.transport.ssl.truststore.secure_password"; - private static final String MAIN_CLASS_NAME = "org.elasticsearch.common.settings.KeyStoreCli"; + private static final String MAIN_CLASS_NAME = "org.elasticsearch.launcher.CliToolLauncher"; private final Map properties = new LinkedHashMap<>(); private final JavaCommand command; private EsKeyStoreCli(EsInstallation esInstallation) { String esHomeAbsolutePath = esInstallation.getHomeDirectory().getAbsolutePath(); - command = new JavaCommand(ProcessId.ELASTICSEARCH, esInstallation.getConfDirectory()) + command = new JavaCommand(ProcessId.ELASTICSEARCH, esInstallation.getHomeDirectory()) .setClassName(MAIN_CLASS_NAME) .setJvmOptions(new EsKeyStoreJvmOptions(esInstallation)) - .addClasspath(Paths.get(esHomeAbsolutePath, "lib").toAbsolutePath() + "/*") - .addClasspath(Paths.get(esHomeAbsolutePath, "lib", "tools", "keystore-cli") - .toAbsolutePath() + "/*") + .addClasspath(Paths.get(esHomeAbsolutePath, "lib", "").toAbsolutePath() + File.separator + "*") + .addClasspath(Paths.get(esHomeAbsolutePath, "lib", "cli-launcher", "").toAbsolutePath() + File.separator + "*") .addParameter("add") .addParameter("-x") .addParameter("-f"); @@ -118,13 +118,14 @@ public class EsKeyStoreCli { private static Map mandatoryOptions(EsInstallation esInstallation) { Map res = new LinkedHashMap<>(7); - res.put("-Xshare:auto", ""); res.put("-Xms4m", ""); res.put("-Xmx64m", ""); + res.put("-XX:+UseSerialGC", ""); + res.put("-Dcli.name=", ""); + res.put("-Dcli.script=", "bin/elasticsearch-keystore"); + res.put("-Dcli.libs=", "lib/tools/keystore-cli"); res.put("-Des.path.home=", esInstallation.getHomeDirectory().getAbsolutePath()); res.put("-Des.path.conf=", esInstallation.getConfDirectory().getAbsolutePath()); - res.put("-Des.distribution=", "default"); - res.put("-Des.distribution.type=", "tar"); return res; } } diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java index cfb20a2a39a..843fd1e3585 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java @@ -81,7 +81,7 @@ public class EsSettings { this.clusterName = props.nonNullValue(CLUSTER_NAME.getKey()); this.clusterEnabled = props.valueAsBoolean(CLUSTER_ENABLED.getKey()); if (this.clusterEnabled) { - this.nodeName = props.value(CLUSTER_NODE_NAME.getKey(), "sonarqube-" + UUID.randomUUID().toString()); + this.nodeName = props.value(CLUSTER_NODE_NAME.getKey(), "sonarqube-" + UUID.randomUUID()); } else { this.nodeName = STANDALONE_NODE_NAME; } @@ -123,6 +123,9 @@ public class EsSettings { builder.put("xpack.security.transport.ssl.verification_mode", "certificate"); builder.put("xpack.security.transport.ssl.keystore.path", clusterESKeystoreFileName); builder.put("xpack.security.transport.ssl.truststore.path", clusterESTruststoreFileName); + } else { + builder.put("xpack.security.autoconfiguration.enabled", Boolean.FALSE.toString()); + builder.put("xpack.security.enabled", Boolean.FALSE.toString()); } } @@ -151,8 +154,7 @@ public class EsSettings { int searchPort = Integer.parseInt(props.nonNullValue(SEARCH_PORT.getKey())); builder.put(ES_HTTP_HOST_KEY, searchHost.getHostAddress()); builder.put(ES_HTTP_PORT_KEY, valueOf(searchPort)); - builder.put("discovery.seed_hosts", searchHost.getHostAddress()); - builder.put("cluster.initial_master_nodes", searchHost.getHostAddress()); + builder.put("discovery.type", "single-node"); int transportPort = Integer.parseInt(props.nonNullValue(ES_PORT.getKey())); @@ -225,22 +227,20 @@ public class EsSettings { builder.put("cluster.routing.allocation.awareness.attributes", "rack_id"); builder.put("node.attr.rack_id", nodeName); builder.put("node.name", nodeName); - builder.put("node.data", valueOf(true)); - builder.put("node.master", valueOf(true)); } private void configureOthers(Map builder) { builder.put("action.auto_create_index", String.valueOf(false)); - if (props.value(JAVA_ADDITIONAL_OPS_PROPERTY, "").contains("-D" + SECCOMP_PROPERTY + "=false")) { - builder.put(SECCOMP_PROPERTY, "false"); + if (props.value(JAVA_ADDITIONAL_OPS_PROPERTY, "").contains("-D" + SECCOMP_PROPERTY + "=" + Boolean.FALSE)) { + builder.put(SECCOMP_PROPERTY, Boolean.FALSE.toString()); } - if (props.value(JAVA_ADDITIONAL_OPS_PROPERTY, "").contains("-Dnode.store.allow_mmapfs=false")) { + if (props.value(JAVA_ADDITIONAL_OPS_PROPERTY, "").contains("-Dnode.store.allow_mmapfs=" + Boolean.FALSE)) { throw new MessageException("Property 'node.store.allow_mmapfs' is no longer supported. Use 'node.store.allow_mmap' instead."); } - if (props.value(JAVA_ADDITIONAL_OPS_PROPERTY, "").contains("-D" + ALLOW_MMAP + "=false")) { - builder.put(ALLOW_MMAP, "false"); + if (props.value(JAVA_ADDITIONAL_OPS_PROPERTY, "").contains("-D" + ALLOW_MMAP + "=" + Boolean.FALSE)) { + builder.put(ALLOW_MMAP, Boolean.FALSE.toString()); } } } diff --git a/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java index db5f8efd230..2700d5ef0b2 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java @@ -32,7 +32,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.application.command.EsJvmOptions; -import org.sonar.application.command.EsScriptCommand; import org.sonar.application.command.JavaCommand; import org.sonar.application.command.JvmOptions; import org.sonar.application.es.EsInstallation; @@ -239,7 +238,7 @@ public class ProcessLauncherImplTest { File dataDir = temp.newFolder(); File logDir = temp.newFolder(); ProcessLauncher underTest = new ProcessLauncherImpl(tempDir, commands, TestProcessBuilder::new); - EsScriptCommand command = createEsScriptCommand(tempDir, homeDir, dataDir, logDir); + JavaCommand command = createEsCommand(tempDir, homeDir, dataDir, logDir); File outdatedEsDir = new File(dataDir, "es"); assertThat(outdatedEsDir.mkdir()).isTrue(); @@ -257,7 +256,7 @@ public class ProcessLauncherImplTest { File dataDir = temp.newFolder(); File logDir = temp.newFolder(); ProcessLauncher underTest = new ProcessLauncherImpl(tempDir, commands, TestProcessBuilder::new); - EsScriptCommand command = createEsScriptCommand(tempDir, homeDir, dataDir, logDir); + JavaCommand command = createEsCommand(tempDir, homeDir, dataDir, logDir); File outdatedEsDir = new File(dataDir, "es"); assertThat(outdatedEsDir).doesNotExist(); @@ -280,13 +279,14 @@ public class ProcessLauncherImplTest { .hasMessage("Fail to launch process [%s]", ProcessId.ELASTICSEARCH.getHumanReadableName()); } - private EsScriptCommand createEsScriptCommand(File tempDir, File homeDir, File dataDir, File logDir) throws IOException { - EsScriptCommand command = new EsScriptCommand(ProcessId.ELASTICSEARCH, temp.newFolder()); + private JavaCommand createEsCommand(File tempDir, File homeDir, File dataDir, File logDir) throws IOException { + JavaCommand command = new JavaCommand(ProcessId.ELASTICSEARCH, temp.newFolder()); Props props = new Props(new Properties()); props.set("sonar.path.temp", tempDir.getAbsolutePath()); props.set("sonar.path.home", homeDir.getAbsolutePath()); props.set("sonar.path.data", dataDir.getAbsolutePath()); props.set("sonar.path.logs", logDir.getAbsolutePath()); + command.setJvmOptions(mock(JvmOptions.class)); command.setEsInstallation(new EsInstallation(props) .setEsYmlSettings(mock(EsYmlSettings.class)) .setEsJvmOptions(mock(EsJvmOptions.class)) diff --git a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java index eb7bf1fd24a..18b19214723 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java @@ -43,7 +43,6 @@ import org.mockito.Mockito; import org.slf4j.LoggerFactory; import org.sonar.application.command.AbstractCommand; import org.sonar.application.command.CommandFactory; -import org.sonar.application.command.EsScriptCommand; import org.sonar.application.command.JavaCommand; import org.sonar.application.config.AppSettings; import org.sonar.application.config.TestAppSettings; @@ -75,7 +74,7 @@ public class SchedulerImplTest { private Level initialLevel; - private EsScriptCommand esScriptCommand; + private JavaCommand esCommand; private JavaCommand webLeaderCommand; private JavaCommand webFollowerCommand; private JavaCommand ceCommand; @@ -91,7 +90,7 @@ public class SchedulerImplTest { @Before public void setUp() throws Exception { File tempDir = temporaryFolder.newFolder(); - esScriptCommand = new EsScriptCommand(ELASTICSEARCH, tempDir); + esCommand = new JavaCommand(ELASTICSEARCH, tempDir); webLeaderCommand = new JavaCommand(WEB_SERVER, tempDir); webFollowerCommand = new JavaCommand(WEB_SERVER, tempDir); ceCommand = new JavaCommand(COMPUTE_ENGINE, tempDir); @@ -348,7 +347,7 @@ public class SchedulerImplTest { TestManagedProcess es = processLauncher.waitForProcess(ELASTICSEARCH); assertThat(es.isAlive()).isTrue(); assertThat(processLauncher.processes).hasSize(1); - assertThat(processLauncher.commands).containsExactly(esScriptCommand); + assertThat(processLauncher.commands).containsExactly(esCommand); // elasticsearch becomes operational -> web leader is starting es.signalAsOperational(); @@ -356,7 +355,7 @@ public class SchedulerImplTest { TestManagedProcess web = processLauncher.waitForProcess(WEB_SERVER); assertThat(web.isAlive()).isTrue(); assertThat(processLauncher.processes).hasSize(2); - assertThat(processLauncher.commands).containsExactly(esScriptCommand, webLeaderCommand); + assertThat(processLauncher.commands).containsExactly(esCommand, webLeaderCommand); // web becomes operational -> CE is starting web.signalAsOperational(); @@ -364,7 +363,7 @@ public class SchedulerImplTest { TestManagedProcess ce = processLauncher.waitForProcess(COMPUTE_ENGINE).signalAsOperational(); assertThat(ce.isAlive()).isTrue(); assertThat(processLauncher.processes).hasSize(3); - assertThat(processLauncher.commands).containsExactly(esScriptCommand, webLeaderCommand, ceCommand); + assertThat(processLauncher.commands).containsExactly(esCommand, webLeaderCommand, ceCommand); // all processes are up processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isTrue()); @@ -402,8 +401,8 @@ public class SchedulerImplTest { private class TestCommandFactory implements CommandFactory { @Override - public EsScriptCommand createEsCommand() { - return esScriptCommand; + public JavaCommand createEsCommand() { + return esCommand; } @Override diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java index 638164f3211..dad21241b8e 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java @@ -40,7 +40,6 @@ import org.sonar.process.Props; import org.sonar.process.System2; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.entry; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -107,13 +106,6 @@ public class CommandFactoryImplTest { "Use properties sonar.search.javaOpts and/or sonar.search.javaAdditionalOpts in sonar.properties to change SQ JVM processes options"); } - @Test - public void createEsCommand_throws_ISE_if_es_binary_is_not_found() { - assertThatThrownBy(() -> newFactory(new Properties()).createEsCommand()) - .isInstanceOf(IllegalStateException.class) - .hasMessage("Cannot find elasticsearch binary"); - } - @Test public void createEsCommand_for_unix_returns_command_for_default_settings() throws Exception { when(system2.isOsWindows()).thenReturn(false); @@ -122,9 +114,7 @@ public class CommandFactoryImplTest { Properties props = new Properties(); props.setProperty("sonar.search.host", "localhost"); - AbstractCommand command = newFactory(props, system2).createEsCommand(); - assertThat(command).isInstanceOf(EsScriptCommand.class); - EsScriptCommand esCommand = (EsScriptCommand) command; + JavaCommand esCommand = newFactory(props, system2).createEsCommand(); EsInstallation esConfig = esCommand.getEsInstallation(); assertThat(esConfig.getHost()).isNotEmpty(); @@ -139,57 +129,13 @@ public class CommandFactoryImplTest { assertThat(esConfig.getLog4j2Properties()) .contains(entry("appender.file_es.fileName", new File(logsDir, "es.log").getAbsolutePath())); - File esConfDir = new File(tempDir, "conf/es"); assertThat(esCommand.getEnvVariables()) - .contains(entry("ES_PATH_CONF", esConfDir.getAbsolutePath())) - .contains(entry("ES_JVM_OPTIONS", new File(esConfDir, "jvm.options").getAbsolutePath())) .containsKey("ES_JAVA_HOME"); - assertThat(esCommand.getSuppressedEnvVariables()).containsOnly("JAVA_TOOL_OPTIONS", "ES_JAVA_OPTS"); - - assertThat(esConfig.getEsJvmOptions().getAll()) - .contains("-Djava.io.tmpdir=" + tempDir.getAbsolutePath()); - } - - @Test - public void createEsCommand_for_windows_returns_command_for_default_settings() throws Exception { - when(system2.isOsWindows()).thenReturn(true); - prepareEsFileSystem(); - - Properties props = new Properties(); - props.setProperty("sonar.search.host", "localhost"); - AbstractCommand command = newFactory(props, system2).createEsCommand(); - assertThat(command).isInstanceOf(JavaCommand.class); - JavaCommand esCommand = (JavaCommand) command; - EsInstallation esConfig = esCommand.getEsInstallation(); - - assertThat(esConfig.getHost()).isNotEmpty(); - assertThat(esConfig.getHttpPort()).isEqualTo(9001); - assertThat(esConfig.getEsJvmOptions().getAll()) - // enforced values - .contains("-XX:+UseG1GC") - .contains("-Dfile.encoding=UTF-8") - // default settings - .contains("-Xms512m", "-Xmx512m", "-XX:+HeapDumpOnOutOfMemoryError"); - assertThat(esConfig.getEsYmlSettings()).isNotNull(); - - assertThat(esConfig.getLog4j2Properties()) - .contains(entry("appender.file_es.fileName", new File(logsDir, "es.log").getAbsolutePath())); - - File esConfDir = new File(tempDir, "conf/es"); - assertThat(esCommand.getArguments()).isEmpty(); - assertThat(esCommand.getEnvVariables()) - .contains(entry("ES_JVM_OPTIONS", new File(esConfDir, "jvm.options").getAbsolutePath())) - .containsKey("ES_JAVA_HOME"); - assertThat(esCommand.getSuppressedEnvVariables()).containsOnly("JAVA_TOOL_OPTIONS", "ES_JAVA_OPTS"); - - assertThat(esConfig.getEsJvmOptions().getAll()) - .contains("-Djava.io.tmpdir=" + tempDir.getAbsolutePath()); assertThat(esCommand.getJvmOptions().getAll()) - .containsAll(esConfig.getEsJvmOptions().getAll()) - .contains("-Delasticsearch") - .contains("-Des.path.home=" + new File(homeDir, "elasticsearch")) - .contains("-Des.path.conf=" + esConfDir.getAbsolutePath()); + .contains("-Xms4m", "-Xmx64m", "-XX:+UseSerialGC", "-Dcli.name=server", "-Dcli.script=./bin/elasticsearch", + "-Dcli.libs=lib/tools/server-cli", "-Des.path.home=" + esConfig.getHomeDirectory().getAbsolutePath(), + "-Des.path.conf=" + esConfig.getConfDirectory().getAbsolutePath(), "-Des.distribution.type=tar"); } @Test diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/EsJvmOptionsTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/EsJvmOptionsTest.java index 2de7f134905..fbc43dbf721 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/command/EsJvmOptionsTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/command/EsJvmOptionsTest.java @@ -75,7 +75,8 @@ public class EsJvmOptionsTest { "-Dlog4j2.formatMsgNoLookups=true", "-Djava.locale.providers=COMPAT", "-Dcom.redhat.fips=false", - "-Des.enforce.bootstrap.checks=true"); + "-Des.enforce.bootstrap.checks=true", + "-Xlog:disable"); } @Test @@ -193,6 +194,7 @@ public class EsJvmOptionsTest { "-XX:+UseG1GC\n" + "-Djava.io.tmpdir=" + tmpDir.getAbsolutePath() + "\n" + "-XX:ErrorFile=" + Paths.get("path_to_logs/es_hs_err_pid%p.log").toAbsolutePath() + "\n" + + "-Xlog:disable\n" + "-Des.networkaddress.cache.ttl=60\n" + "-Des.networkaddress.cache.negative.ttl=10\n" + "-XX:+AlwaysPreTouch\n" + diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsInstallationTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsInstallationTest.java index c7587013754..49734dc582a 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsInstallationTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsInstallationTest.java @@ -97,7 +97,7 @@ public class EsInstallationTest { EsInstallation underTest = new EsInstallation(props); - assertThat(underTest.getDataDirectory()).isEqualTo(new File(dataDir, "es7")); + assertThat(underTest.getDataDirectory()).isEqualTo(new File(dataDir, "es8")); } @Test @@ -129,7 +129,7 @@ public class EsInstallationTest { assertThat(underTest.getOutdatedSearchDirectories()) .extracting(File::getName) - .containsExactlyInAnyOrder("es", "es5", "es6"); + .containsExactlyInAnyOrder("es", "es5", "es6", "es7"); } @Test diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsKeyStoreCliTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsKeyStoreCliTest.java index 36a0abade5f..ea0f2507bbc 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsKeyStoreCliTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsKeyStoreCliTest.java @@ -63,21 +63,22 @@ public class EsKeyStoreCliTest { JavaCommand executedCommand = process.getExecutedCommand(); - String expectedHomeLibPath = Paths.get(homeDir.toString(), "lib") + "/*"; - String expectedHomeKeystorePath = Paths.get(homeDir.toString(), "lib", "tools", "keystore-cli") + "/*"; + String expectedHomeLibPath = Paths.get(homeDir.toString(), "lib", "*").toString(); + String expectedHomeKeystorePath = Paths.get(homeDir.toString(), "lib", "cli-launcher", "*").toString(); - assertThat(executedCommand.getClassName()).isEqualTo("org.elasticsearch.common.settings.KeyStoreCli"); + assertThat(executedCommand.getClassName()).isEqualTo("org.elasticsearch.launcher.CliToolLauncher"); assertThat(executedCommand.getClasspath()) .containsExactly(expectedHomeLibPath, expectedHomeKeystorePath); assertThat(executedCommand.getParameters()).containsExactly("add", "-x", "-f", "test.property1", "test.property2", "test.property3"); assertThat(executedCommand.getJvmOptions().getAll()).containsExactly( - "-Xshare:auto", "-Xms4m", "-Xmx64m", + "-XX:+UseSerialGC", + "-Dcli.name=", + "-Dcli.script=bin/elasticsearch-keystore", + "-Dcli.libs=lib/tools/keystore-cli", "-Des.path.home=" + homeDir.toPath(), - "-Des.path.conf=" + confDir.toPath(), - "-Des.distribution=default", - "-Des.distribution.type=tar"); + "-Des.path.conf=" + confDir.toPath()); verify(process.getOutputStream()).write(argThat(new ArrayContainsMatcher("value1\nvalue2\nvalue3\n")), eq(0), eq(21)); verify(process.getOutputStream(), atLeastOnce()).flush(); diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java index 54711558f9c..fb66bce27df 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java @@ -154,8 +154,9 @@ public class EsSettingsTest { // no cluster, but cluster and node names are set though .containsEntry("cluster.name", "sonarqube") .containsEntry("node.name", "sonarqube") - .containsEntry("discovery.seed_hosts", "127.0.0.1") - .containsEntry("cluster.initial_master_nodes", "127.0.0.1"); + .containsEntry("discovery.type", "single-node") + .doesNotContainKey("discovery.seed_hosts") + .doesNotContainKey("cluster.initial_master_nodes"); assertThat(generated.get("path.data")).isNotNull(); assertThat(generated.get("path.logs")).isNotNull(); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java index e7d089f4719..78a5a666de1 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java @@ -164,15 +164,8 @@ public abstract class BaseDoc { .source(getFields()); } - public static long epochMillisToEpochSeconds(long epochMillis) { - return epochMillis / 1000L; - } - private static Date epochSecondsToDate(Number value) { return new Date(value.longValue() * 1000L); } - public static long dateToEpochSeconds(Date date) { - return epochMillisToEpochSeconds(date.getTime()); - } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java index 3fe2862f5e2..ca881091fad 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java @@ -266,11 +266,11 @@ public class EsClient implements Closeable { private static final int SOCKET_TIMEOUT = 60000; public MinimalRestHighLevelClient(@Nullable String searchPassword, HttpHost... hosts) { - super(buildHttpClient(searchPassword, hosts)); + super(buildHttpClient(searchPassword, hosts).build(), RestClient::close, Lists.newArrayList(), true); } MinimalRestHighLevelClient(RestClient restClient) { - super(restClient, RestClient::close, Lists.newArrayList()); + super(restClient, RestClient::close, Lists.newArrayList(), true); } @NotNull diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/FieldAware.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/FieldAware.java index b903328b964..06ab86f370d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/FieldAware.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/FieldAware.java @@ -67,7 +67,7 @@ public abstract class FieldAware> { public U createDateTimeField(String fieldName) { Map hash = new TreeMap<>(); hash.put("type", "date"); - hash.put("format", "date_time||epoch_second"); + hash.put("format", "date_time||epoch_millis"); return setField(fieldName, hash); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java index 08dac287a6e..d824901b263 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java @@ -124,10 +124,9 @@ public class RuleIndexDefinition implements IndexDefinition { ruleMapping.keywordFieldBuilder(FIELD_RULE_INTERNAL_KEY).disableNorms().disableSearch().build(); ruleMapping.keywordFieldBuilder(FIELD_RULE_NAME).addSubFields(SORTABLE_ANALYZER, SEARCH_GRAMS_ANALYZER).build(); - ruleMapping.keywordFieldBuilder(FIELD_RULE_HTML_DESCRIPTION) + ruleMapping.textFieldBuilder(FIELD_RULE_HTML_DESCRIPTION) .disableSearch() .disableNorms() - .disableSortingAndAggregating() .addSubFields(ENGLISH_HTML_ANALYZER) .build(); ruleMapping.keywordFieldBuilder(FIELD_RULE_SEVERITY).disableNorms().build(); diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 56f309a48d0..a19c097a09c 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -73,7 +73,6 @@ import org.sonar.api.server.rule.RulesDefinition.PciDssVersion; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.server.es.BaseDoc; import org.sonar.server.es.EsClient; import org.sonar.server.es.EsUtils; import org.sonar.server.es.SearchOptions; @@ -109,7 +108,6 @@ import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.sonar.api.rules.RuleType.SECURITY_HOTSPOT; import static org.sonar.api.rules.RuleType.VULNERABILITY; import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; -import static org.sonar.server.es.BaseDoc.epochMillisToEpochSeconds; import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; import static org.sonar.server.es.searchrequest.TopAggregationDefinition.NON_STICKY; @@ -718,20 +716,20 @@ public class IssueIndex { "__createdAfter", CREATED_AT.getFilterScope(), QueryBuilders .rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT) - .from(BaseDoc.dateToEpochSeconds(createdAfter.date()), createdAfter.inclusive())); + .from(createdAfter.date().getTime(), createdAfter.inclusive())); } if (createdBefore != null) { filters.addFilter( "__createdBefore", CREATED_AT.getFilterScope(), QueryBuilders .rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT) - .lt(BaseDoc.dateToEpochSeconds(createdBefore))); + .lt(createdBefore.getTime())); } Date createdAt = query.createdAt(); if (createdAt != null) { filters.addFilter( "__createdAt", CREATED_AT.getFilterScope(), - termQuery(FIELD_ISSUE_FUNC_CREATED_AT, BaseDoc.dateToEpochSeconds(createdAt))); + termQuery(FIELD_ISSUE_FUNC_CREATED_AT, createdAt.getTime())); } } @@ -765,7 +763,7 @@ public class IssueIndex { BoolQueryBuilder boolQueryBuilder = boolQuery(); createdAfterByProjectUuids.forEach((projectOrProjectBranchUuid, createdAfterDate) -> boolQueryBuilder.should(boolQuery() .filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectOrProjectBranchUuid)) - .filter(rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT).from(BaseDoc.dateToEpochSeconds(createdAfterDate.date()), createdAfterDate.inclusive())))); + .filter(rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT).from(createdAfterDate.date().getTime(), createdAfterDate.inclusive())))); allFilters.addFilter("__created_after_by_project_uuids", new SimpleFieldFilterScope("createdAfterByProjectUuids"), boolQueryBuilder); } @@ -1067,7 +1065,7 @@ public class IssueIndex { .aggregation(AggregationBuilders .filter(projectUuid, boolQuery() .filter(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid)) - .filter(rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT).gte(epochMillisToEpochSeconds(from)))) + .filter(rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT).gte(from))) .subAggregation( AggregationBuilders.terms("branchUuid").field(FIELD_ISSUE_BRANCH_UUID) .subAggregation( diff --git a/sonar-application/build.gradle b/sonar-application/build.gradle index 3a4782028ef..d6da1f90227 100644 --- a/sonar-application/build.gradle +++ b/sonar-application/build.gradle @@ -91,23 +91,31 @@ task verifySonarProperties(type: Verify) { } } +def artifactoryUsername = System.env.'ARTIFACTORY_PRIVATE_USERNAME' ?: (project.hasProperty('artifactoryUsername') ? project.getProperty('artifactoryUsername') : '') +def artifactoryPassword = System.env.'ARTIFACTORY_PRIVATE_PASSWORD' ?: (project.hasProperty('artifactoryPassword') ? project.getProperty('artifactoryPassword') : '') + task verifyElasticSearchDownload(type: Verify) { - src new File(buildDir, "$elasticsearchDownloadUrlFile") - algorithm 'SHA-512' - checksum elasticsearchDownloadSha512 + if (artifactoryUsername && artifactoryPassword) { + src new File(buildDir, "$elasticsearchDownloadUrlFileNoJdk") + algorithm 'SHA-512' + checksum elasticsearchDownloadSha512NoJdk + } else { + src new File(buildDir, "$elasticsearchDownloadUrlFileJdk") + algorithm 'SHA-512' + checksum elasticsearchDownloadSha512Jdk + } } task downloadElasticSearch(type: Download) { - def artifactoryUsername = System.env.'ARTIFACTORY_PRIVATE_USERNAME' ?: (project.hasProperty('artifactoryUsername') ? project.getProperty('artifactoryUsername') : '') - def artifactoryPassword = System.env.'ARTIFACTORY_PRIVATE_PASSWORD' ?: (project.hasProperty('artifactoryPassword') ? project.getProperty('artifactoryPassword') : '') if (artifactoryUsername && artifactoryPassword) { - src "$elasticsearchDownloadRepoxUrlPath$elasticsearchDownloadUrlFile" + src "$elasticsearchDownloadRepoxUrlPath$elasticsearchDownloadUrlFileNoJdk" username artifactoryUsername password artifactoryPassword + dest "$buildDir/$elasticsearchDownloadUrlFileNoJdk" } else { - src "$elasticsearchDownloadUrlPath$elasticsearchDownloadUrlFile" + src "$elasticsearchDownloadUrlPath$elasticsearchDownloadUrlFileJdk" + dest "$buildDir/$elasticsearchDownloadUrlFileJdk" } - dest "$buildDir/$elasticsearchDownloadUrlFile" onlyIfModified true finalizedBy verifyElasticSearchDownload } @@ -148,7 +156,6 @@ task zip(type: Zip, dependsOn: [configurations.compileClasspath, downloadElastic exclude 'conf/sonar.properties' exclude 'bin/windows-x86-64/lib/SonarServiceWrapperTemplate.xml' exclude 'bin/windows-x86-64/StartSonar.bat' - exclude 'elasticsearch-patch' exclude 'bin/linux-x86-64/sonar.sh' exclude 'bin/macosx-universal-64/sonar.sh' } @@ -161,36 +168,16 @@ task zip(type: Zip, dependsOn: [configurations.compileClasspath, downloadElastic fcd.relativePath = new RelativePath(true, *path) } into("${archiveDir}/elasticsearch") - filesMatching('**/bin/elasticsearch-env') { - // to avoid warning logs - filter { line -> line.replaceAll('echo "warning: no-jdk distributions.*', ':') } - } - // elasticsearch script will be replaced by patched version below - exclude '**/bin/elasticsearch' - exclude '**/bin/elasticsearch-certgen' - exclude '**/bin/elasticsearch-certutil' - exclude '**/bin/elasticsearch-cli' - exclude '**/bin/elasticsearch-croneval' - exclude '**/bin/elasticsearch-geoip' - exclude '**/bin/elasticsearch-keystore' - exclude '**/bin/elasticsearch-migrate' - exclude '**/bin/elasticsearch-node' - exclude '**/bin/elasticsearch-plugin' - exclude '**/bin/elasticsearch-saml-metadata' - exclude '**/bin/elasticsearch-service-tokens' - exclude '**/bin/elasticsearch-setup-passwords' - exclude '**/bin/elasticsearch-shard' - exclude '**/bin/elasticsearch-sql-cli*' - exclude '**/bin/elasticsearch-syskeygen' - exclude '**/bin/elasticsearch-users' - exclude '**/bin/x-pack-env' - exclude '**/bin/x-pack-security-env' - exclude '**/bin/x-pack-watcher-env' + exclude '**/bin/**' exclude '**/jdk/**' - exclude '**/lib/tools/**' + exclude '**/lib/tools/plugin-cli' + exclude '**/lib/tools/geoip-cli' + exclude '**/lib/tools/ansi-console' exclude '**/modules/aggs-matrix-stats/**' exclude '**/modules/constant-keyword/**' + exclude '**/modules/data-streams/**' exclude '**/modules/frozen-indices/**' + exclude '**/modules/ingest-attachment/**' exclude '**/modules/ingest-common/**' exclude '**/modules/ingest-geoip/**' exclude '**/modules/ingest-user-agent/**' @@ -203,11 +190,15 @@ task zip(type: Zip, dependsOn: [configurations.compileClasspath, downloadElastic exclude '**/modules/percolator/**' exclude '**/modules/rank-eval/**' exclude '**/modules/repositories-metering-api/**' + exclude '**/modules/repository-azure/**' exclude '**/modules/repository-encrypted/**' + exclude '**/modules/repository-gcs/**' + exclude '**/modules/repository-s3/**' exclude '**/modules/repository-url/**' exclude '**/modules/runtime-fields-common/**' exclude '**/modules/search-business-rules/**' exclude '**/modules/searchable-snapshots/**' + exclude '**/modules/snapshot-based-recoveries/**' exclude '**/modules/snapshot-repo-test-kit/**' exclude '**/modules/spatial/**' exclude '**/modules/transform/**' @@ -215,7 +206,31 @@ task zip(type: Zip, dependsOn: [configurations.compileClasspath, downloadElastic exclude '**/modules/vector-tile/**' exclude '**/modules/vectors/**' exclude '**/modules/wildcard/**' - exclude '**/modules/x-pack-*/**' + exclude '**/modules/x-pack-aggregate-metric' + exclude '**/modules/x-pack-analytics/**' + exclude '**/modules/x-pack-async/**' + exclude '**/modules/x-pack-async-search/**' + exclude '**/modules/x-pack-autoscaling/**' + exclude '**/modules/x-pack-ccr/**' + exclude '**/modules/x-pack-deprecation/**' + exclude '**/modules/x-pack-enrich/**' + exclude '**/modules/x-pack-eql/**' + exclude '**/modules/x-pack-fleet/**' + exclude '**/modules/x-pack-graph/**' + exclude '**/modules/x-pack-identity-provider/**' + exclude '**/modules/x-pack-ilm/**' + exclude '**/modules/x-pack-logstash/**' + exclude '**/modules/x-pack-monitoring/**' + exclude '**/modules/x-pack-ql/**' + exclude '**/modules/x-pack-rollup/**' + exclude '**/modules/x-pack-ml/**' + exclude '**/modules/x-pack-shutdown/**' + exclude '**/modules/x-pack-sql/**' + exclude '**/modules/x-pack-stack/**' + exclude '**/modules/x-pack-text-structure/**' + exclude '**/modules/x-pack-voting-only-node/**' + exclude '**/modules/x-pack-watcher/**' + exclude '**/modules/x-pack-write-load-forecaster/**' includeEmptyDirs = false } @@ -258,11 +273,6 @@ task zip(type: Zip, dependsOn: [configurations.compileClasspath, downloadElastic 'sqversion': version ]) } - - into("${archiveDir}/elasticsearch/") { - from file('src/main/assembly/elasticsearch-patch') - include 'bin/elasticsearch' - } // Create the empty dir (plugins) required by elasticsearch into("${archiveDir}/elasticsearch/") { from "$buildDir/elasticsearch" @@ -311,8 +321,8 @@ zip.doFirst { } // Check the size of the archive zip.doLast { - def minLength = 272000000 - def maxLength = 297000000 + def minLength = 320000000 + def maxLength = 345000000 def length = archiveFile.get().asFile.length() if (length < minLength) diff --git a/sonar-application/src/main/assembly/elasticsearch-patch/bin/elasticsearch b/sonar-application/src/main/assembly/elasticsearch-patch/bin/elasticsearch deleted file mode 100755 index 704f7745fef..00000000000 --- a/sonar-application/src/main/assembly/elasticsearch-patch/bin/elasticsearch +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -# CONTROLLING STARTUP: -# -# This script relies on a few environment variables to determine startup -# behavior, those variables are: -# -# ES_PATH_CONF -- Path to config directory -# ES_JAVA_OPTS -- External Java Opts on top of the defaults set -# -# Optionally, exact memory values can be set using the `ES_JAVA_OPTS`. Note that -# the Xms and Xmx lines in the JVM options file must be commented out. Example -# values are "512m", and "10g". -# -# ES_JAVA_OPTS="-Xms8g -Xmx8g" ./bin/elasticsearch - -source "`dirname "$0"`"/elasticsearch-env - -ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options -#### SQ WORKAROUND https://github.com/elastic/elasticsearch/issues/39965 #### -# ES relies on JvmOptionsParser to apply Java version specific options. SQ does not need that because it generates -# the content of the jvm.options file depending on Java version already. -# -# Also, ES accepts extra JVM options specified with ES_JAVA_OPTS env variable. SQ silences this env variable to enforce -# end users specify ES JVM options with "sonar.search.javaOpts" and "sonar.search.javaAdditionalOpt". Workaround drops -# supports for ES_JAVA_OPTS as it just make things simpler. -# -# Below we read content of file "$ES_JVM_OPTIONS" directly in bash, ignoring comments and empty lines and build -# a JVM_OPTIONS array. We later use "${JVM_OPTIONS[@]}" instead of $ES_JAVA_OPTS to benefit from built-in escaping of -# each array value when building the Java command line. -# -# Note: "|| [[ "$line" ]]" below is required to support jvm.options file not ending with an empty line -JVM_OPTIONS=() -while read -r line || [[ "$line" ]]; do - # ignore empty or comment lines - if [[ -z "$line" ]] || [[ $line =~ ^#.* ]]; then - continue - fi - JVM_OPTIONS+=("$line") -done < "$ES_JVM_OPTIONS" - -cd "$ES_HOME" -# manual parsing to find out, if process should be detached -if ! echo $* | grep -E '(^-d |-d$| -d |--daemonize$|--daemonize )' > /dev/null; then - exec \ - "$JAVA" \ - "${JVM_OPTIONS[@]}" \ - -Des.path.home="$ES_HOME" \ - -Des.path.conf="$ES_PATH_CONF" \ - -Des.distribution.flavor="$ES_DISTRIBUTION_FLAVOR" \ - -Des.distribution.type="$ES_DISTRIBUTION_TYPE" \ - -Des.bundled_jdk="false" \ - -cp "$ES_CLASSPATH" \ - org.elasticsearch.bootstrap.Elasticsearch \ - "$@" -else - exec \ - "$JAVA" \ - "${JVM_OPTIONS[@]}" \ - -Des.path.home="$ES_HOME" \ - -Des.path.conf="$ES_PATH_CONF" \ - -Des.distribution.flavor="$ES_DISTRIBUTION_FLAVOR" \ - -Des.distribution.type="$ES_DISTRIBUTION_TYPE" \ - -Des.bundled_jdk="false" \ - -cp "$ES_CLASSPATH" \ - org.elasticsearch.bootstrap.Elasticsearch \ - "$@" \ - <&- & - retval=$? - pid=$! - [ $retval -eq 0 ] || exit $retval - if [ ! -z "$ES_STARTUP_SLEEP_TIME" ]; then - sleep $ES_STARTUP_SLEEP_TIME - fi - if ! ps -p $pid > /dev/null ; then - exit 1 - fi - exit 0 -fi - -exit $? -- 2.39.5