diff options
29 files changed, 575 insertions, 380 deletions
diff --git a/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java b/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java index abe37386b64..038371ee13b 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java @@ -28,9 +28,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonar.application.command.AbstractCommand; import org.sonar.application.command.CommandFactory; -import org.sonar.application.command.EsCommand; -import org.sonar.application.command.JavaCommand; import org.sonar.application.config.AppSettings; import org.sonar.application.config.ClusterSettings; import org.sonar.application.process.Lifecycle; @@ -108,7 +107,7 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi private void tryToStartEs() { SQProcess process = processesById.get(ProcessId.ELASTICSEARCH); if (process != null) { - tryToStartEsProcess(process, commandFactory::createEsCommand); + tryToStartProcess(process, commandFactory::createEsCommand); } } @@ -124,9 +123,9 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi return; } if (appState.isOperational(ProcessId.WEB_SERVER, false)) { - tryToStartJavaProcess(process, () -> commandFactory.createWebCommand(false)); + tryToStartProcess(process, () -> commandFactory.createWebCommand(false)); } else if (appState.tryToLockWebLeader()) { - tryToStartJavaProcess(process, () -> commandFactory.createWebCommand(true)); + tryToStartProcess(process, () -> commandFactory.createWebCommand(true)); } else { Optional<String> leader = appState.getLeaderHostName(); if (leader.isPresent()) { @@ -140,7 +139,7 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi private void tryToStartCe() { SQProcess process = processesById.get(ProcessId.COMPUTE_ENGINE); if (process != null && appState.isOperational(ProcessId.WEB_SERVER, false) && isEsClientStartable()) { - tryToStartJavaProcess(process, commandFactory::createCeCommand); + tryToStartProcess(process, commandFactory::createCeCommand); } } @@ -149,16 +148,9 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi return appState.isOperational(ProcessId.ELASTICSEARCH, requireLocalEs); } - private void tryToStartJavaProcess(SQProcess process, Supplier<JavaCommand> commandSupplier) { + private void tryToStartProcess(SQProcess process, Supplier<AbstractCommand> commandSupplier) { tryToStart(process, () -> { - JavaCommand command = commandSupplier.get(); - return processLauncher.launch(command); - }); - } - - private void tryToStartEsProcess(SQProcess process, Supplier<EsCommand> commandSupplier) { - tryToStart(process, () -> { - EsCommand command = commandSupplier.get(); + AbstractCommand command = commandSupplier.get(); return processLauncher.launch(command); }); } diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/AbstractCommand.java b/server/sonar-main/src/main/java/org/sonar/application/command/AbstractCommand.java index 2da97bb8f8f..d9520742937 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/AbstractCommand.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/AbstractCommand.java @@ -22,11 +22,10 @@ package org.sonar.application.command; import java.io.File; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.Map; -import java.util.Properties; import java.util.Set; -import javax.annotation.Nullable; +import javax.annotation.CheckForNull; +import org.sonar.application.es.EsInstallation; import org.sonar.process.ProcessId; import org.sonar.process.System2; @@ -35,11 +34,10 @@ import static java.util.Objects.requireNonNull; public abstract class AbstractCommand<T extends AbstractCommand> { // unique key among the group of commands to launch private final ProcessId id; - // program arguments - private final Map<String, String> arguments = new LinkedHashMap<>(); private final Map<String, String> envVariables; private final Set<String> suppressedEnvVariables = new HashSet<>(); private final File workDir; + private EsInstallation esInstallation; protected AbstractCommand(ProcessId id, File workDir, System2 system2) { this.id = requireNonNull(id, "ProcessId can't be null"); @@ -60,26 +58,6 @@ public abstract class AbstractCommand<T extends AbstractCommand> { return (T) this; } - public Map<String, String> getArguments() { - return arguments; - } - - public T setArgument(String key, @Nullable String value) { - if (value == null) { - arguments.remove(key); - } else { - arguments.put(key, value); - } - return castThis(); - } - - public T setArguments(Properties args) { - for (Map.Entry<Object, Object> entry : args.entrySet()) { - setArgument(entry.getKey().toString(), entry.getValue() != null ? entry.getValue().toString() : null); - } - return castThis(); - } - public Map<String, String> getEnvVariables() { return envVariables; } @@ -101,4 +79,15 @@ public abstract class AbstractCommand<T extends AbstractCommand> { requireNonNull(value, "value can't be null")); return castThis(); } + + + public T setEsInstallation(EsInstallation esInstallation) { + this.esInstallation = esInstallation; + return castThis(); + } + + @CheckForNull + public EsInstallation getEsInstallation() { + return esInstallation; + } } diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactory.java b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactory.java index e55f364be11..4342d3bf0e4 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactory.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactory.java @@ -21,7 +21,7 @@ package org.sonar.application.command; public interface CommandFactory { - EsCommand createEsCommand(); + AbstractCommand createEsCommand(); JavaCommand createWebCommand(boolean leader); diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java index 0df586aa7f0..3f840eb0131 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java @@ -23,14 +23,14 @@ import java.io.File; import java.util.Map; import java.util.Optional; import org.slf4j.LoggerFactory; +import org.sonar.application.es.EsInstallation; +import org.sonar.application.es.EsLogging; +import org.sonar.application.es.EsSettings; +import org.sonar.application.es.EsYmlSettings; import org.sonar.process.ProcessId; import org.sonar.process.ProcessProperties; import org.sonar.process.Props; import org.sonar.process.System2; -import org.sonar.application.es.EsFileSystem; -import org.sonar.application.es.EsLogging; -import org.sonar.application.es.EsSettings; -import org.sonar.application.es.EsYmlSettings; import static org.sonar.process.ProcessProperties.HTTPS_PROXY_HOST; import static org.sonar.process.ProcessProperties.HTTPS_PROXY_PORT; @@ -54,10 +54,12 @@ public class CommandFactoryImpl implements CommandFactory { private final Props props; private final File tempDir; + private final System2 system2; public CommandFactoryImpl(Props props, File tempDir, System2 system2) { this.props = props; this.tempDir = tempDir; + this.system2 = system2; String javaToolOptions = system2.getenv(ENV_VAR_JAVA_TOOL_OPTIONS); if (javaToolOptions != null && !javaToolOptions.trim().isEmpty()) { LoggerFactory.getLogger(CommandFactoryImpl.class) @@ -67,28 +69,59 @@ public class CommandFactoryImpl implements CommandFactory { } @Override - public EsCommand createEsCommand() { - EsFileSystem esFileSystem = new EsFileSystem(props); - if (!esFileSystem.getExecutable().exists()) { + public AbstractCommand<?> createEsCommand() { + if (system2.isOsWindows()) { + return createEsCommandForWindows(); + } + return createEsCommandForUnix(); + } + + private EsScriptCommand createEsCommandForUnix() { + EsInstallation esInstallation = createEsInstallation(); + return new EsScriptCommand(ProcessId.ELASTICSEARCH, esInstallation.getHomeDirectory()) + .setEsInstallation(esInstallation) + .addOption("-Epath.conf=" + esInstallation.getConfDirectory().getAbsolutePath()) + .setEnvVariable("ES_JVM_OPTIONS", esInstallation.getJvmOptions().getAbsolutePath()) + .setEnvVariable("JAVA_HOME", System.getProperties().getProperty("java.home")) + .suppressEnvVariable(ENV_VAR_JAVA_TOOL_OPTIONS); + } + + private JavaCommand createEsCommandForWindows() { + EsInstallation esInstallation = createEsInstallation(); + return new JavaCommand<EsJvmOptions>(ProcessId.ELASTICSEARCH, esInstallation.getHomeDirectory()) + .setEsInstallation(esInstallation) + .setReadsArgumentsFromFile(false) + .setArgument("path.conf", esInstallation.getConfDirectory().getAbsolutePath()) + .setJvmOptions(new EsJvmOptions() + .addFromMandatoryProperty(props, ProcessProperties.SEARCH_JAVA_OPTS) + .addFromMandatoryProperty(props, ProcessProperties.SEARCH_JAVA_ADDITIONAL_OPTS) + .add("-Delasticsearch") + .add("-Des.path.home=" + esInstallation.getHomeDirectory()) + ) + .setEnvVariable("ES_JVM_OPTIONS", esInstallation.getJvmOptions().getAbsolutePath()) + .setEnvVariable("JAVA_HOME", System.getProperties().getProperty("java.home")) + .setClassName("org.elasticsearch.bootstrap.Elasticsearch") + .addClasspath("lib/*") + .suppressEnvVariable(ENV_VAR_JAVA_TOOL_OPTIONS); + } + + private EsInstallation createEsInstallation() { + EsInstallation esInstallation = new EsInstallation(props); + if (!esInstallation.getExecutable().exists()) { throw new IllegalStateException("Cannot find elasticsearch binary"); } - Map<String, String> settingsMap = new EsSettings(props, esFileSystem, System2.INSTANCE).build(); + Map<String, String> settingsMap = new EsSettings(props, esInstallation, System2.INSTANCE).build(); - return new EsCommand(ProcessId.ELASTICSEARCH, esFileSystem.getHomeDirectory()) - .setFileSystem(esFileSystem) - .setLog4j2Properties(new EsLogging().createProperties(props, esFileSystem.getLogDirectory())) - .setArguments(props.rawProperties()) - .setClusterName(settingsMap.get("cluster.name")) - .setHost(settingsMap.get("network.host")) - .setPort(Integer.valueOf(settingsMap.get("transport.tcp.port"))) - .addEsOption("-Epath.conf=" + esFileSystem.getConfDirectory().getAbsolutePath()) + esInstallation + .setLog4j2Properties(new EsLogging().createProperties(props, esInstallation.getLogDirectory())) .setEsJvmOptions(new EsJvmOptions() .addFromMandatoryProperty(props, ProcessProperties.SEARCH_JAVA_OPTS) .addFromMandatoryProperty(props, ProcessProperties.SEARCH_JAVA_ADDITIONAL_OPTS)) .setEsYmlSettings(new EsYmlSettings(settingsMap)) - .setEnvVariable("ES_JVM_OPTIONS", esFileSystem.getJvmOptions().getAbsolutePath()) - .setEnvVariable("JAVA_HOME", System.getProperties().getProperty("java.home")) - .suppressEnvVariable(ENV_VAR_JAVA_TOOL_OPTIONS); + .setClusterName(settingsMap.get("cluster.name")) + .setHost(settingsMap.get("network.host")) + .setPort(Integer.valueOf(settingsMap.get("transport.tcp.port"))); + return esInstallation; } @Override @@ -101,6 +134,7 @@ public class CommandFactoryImpl implements CommandFactory { addProxyJvmOptions(jvmOptions); JavaCommand<WebJvmOptions> command = new JavaCommand<WebJvmOptions>(ProcessId.WEB_SERVER, homeDir) + .setReadsArgumentsFromFile(true) .setArguments(props.rawProperties()) .setJvmOptions(jvmOptions) // required for logback tomcat valve @@ -127,6 +161,7 @@ public class CommandFactoryImpl implements CommandFactory { addProxyJvmOptions(jvmOptions); JavaCommand<CeJvmOptions> command = new JavaCommand<CeJvmOptions>(ProcessId.COMPUTE_ENGINE, homeDir) + .setReadsArgumentsFromFile(true) .setArguments(props.rawProperties()) .setJvmOptions(jvmOptions) .setClassName("org.sonar.ce.app.CeServer") diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/EsCommand.java b/server/sonar-main/src/main/java/org/sonar/application/command/EsCommand.java deleted file mode 100644 index d44e272132f..00000000000 --- a/server/sonar-main/src/main/java/org/sonar/application/command/EsCommand.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2017 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.application.command; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import org.sonar.application.es.EsFileSystem; -import org.sonar.application.es.EsYmlSettings; -import org.sonar.process.ProcessId; -import org.sonar.process.System2; - -public class EsCommand extends AbstractCommand<EsCommand> { - private EsFileSystem fileSystem; - private String clusterName; - private String host; - private int port; - private Properties log4j2Properties; - private List<String> esOptions = new ArrayList<>(); - private EsJvmOptions esJvmOptions; - private EsYmlSettings esYmlSettings; - - public EsCommand(ProcessId id, File workDir) { - super(id, workDir, System2.INSTANCE); - } - - public EsFileSystem getFileSystem() { - return fileSystem; - } - - public EsCommand setFileSystem(EsFileSystem fileSystem) { - this.fileSystem = fileSystem; - return this; - } - - public String getClusterName() { - return clusterName; - } - - public EsCommand setClusterName(String clusterName) { - this.clusterName = clusterName; - return this; - } - - public String getHost() { - return host; - } - - public EsCommand setHost(String host) { - this.host = host; - return this; - } - - public int getPort() { - return port; - } - - public EsCommand setPort(int port) { - this.port = port; - return this; - } - - public Properties getLog4j2Properties() { - return log4j2Properties; - } - - public EsCommand setLog4j2Properties(Properties log4j2Properties) { - this.log4j2Properties = log4j2Properties; - return this; - } - - public List<String> getEsOptions() { - return esOptions; - } - - public EsCommand addEsOption(String s) { - if (!s.isEmpty()) { - esOptions.add(s); - } - return this; - } - - public EsCommand setEsJvmOptions(EsJvmOptions esJvmOptions) { - this.esJvmOptions = esJvmOptions; - return this; - } - - public EsJvmOptions getEsJvmOptions() { - return esJvmOptions; - } - - public EsCommand setEsYmlSettings(EsYmlSettings esYmlSettings) { - this.esYmlSettings = esYmlSettings; - return this; - } - - public EsYmlSettings getEsYmlSettings() { - return esYmlSettings; - } -} diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/EsScriptCommand.java b/server/sonar-main/src/main/java/org/sonar/application/command/EsScriptCommand.java new file mode 100644 index 00000000000..72e6f09d48d --- /dev/null +++ b/server/sonar-main/src/main/java/org/sonar/application/command/EsScriptCommand.java @@ -0,0 +1,45 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.application.command; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import org.sonar.process.ProcessId; +import org.sonar.process.System2; + +public class EsScriptCommand extends AbstractCommand<EsScriptCommand> { + private List<String> options = new ArrayList<>(); + + public EsScriptCommand(ProcessId id, File workDir) { + super(id, workDir, System2.INSTANCE); + } + + public List<String> getOptions() { + return options; + } + + public EsScriptCommand addOption(String s) { + if (!s.isEmpty()) { + options.add(s); + } + return this; + } +} diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/JavaCommand.java b/server/sonar-main/src/main/java/org/sonar/application/command/JavaCommand.java index 5cee5a95e33..864a9033b19 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/JavaCommand.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/JavaCommand.java @@ -21,16 +21,23 @@ package org.sonar.application.command; import java.io.File; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Properties; +import javax.annotation.Nullable; import org.sonar.process.ProcessId; import org.sonar.process.System2; public class JavaCommand<T extends JvmOptions> extends AbstractCommand<JavaCommand<T>> { + // program arguments + private final Map<String, String> arguments = new LinkedHashMap<>(); // entry point private String className; private JvmOptions<T> jvmOptions; // relative path to JAR files private final List<String> classpath = new ArrayList<>(); + private boolean readsArgumentsFromFile; public JavaCommand(ProcessId id, File workDir) { super(id, workDir, System2.INSTANCE); @@ -64,6 +71,35 @@ public class JavaCommand<T extends JvmOptions> extends AbstractCommand<JavaComma return this; } + public boolean getReadsArgumentsFromFile() { + return readsArgumentsFromFile; + } + + public JavaCommand<T> setReadsArgumentsFromFile(boolean readsArgumentsFromFile) { + this.readsArgumentsFromFile = readsArgumentsFromFile; + return this; + } + + public Map<String, String> getArguments() { + return arguments; + } + + public JavaCommand<T> setArgument(String key, @Nullable String value) { + if (value == null) { + arguments.remove(key); + } else { + arguments.put(key, value); + } + return this; + } + + public JavaCommand<T> setArguments(Properties args) { + for (Map.Entry<Object, Object> entry : args.entrySet()) { + setArgument(entry.getKey().toString(), entry.getValue() != null ? entry.getValue().toString() : null); + } + return this; + } + @Override public String toString() { return "JavaCommand{" + "workDir=" + getWorkDir() + diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsFileSystem.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java index 1fb91dc5287..b7eb3b44d8b 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsFileSystem.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java @@ -22,6 +22,8 @@ package org.sonar.application.es; import java.io.File; import java.util.Collections; import java.util.List; +import java.util.Properties; +import org.sonar.application.command.EsJvmOptions; import org.sonar.process.ProcessProperties; import org.sonar.process.Props; @@ -33,14 +35,20 @@ import org.sonar.process.Props; * This class does not ensure files nor directories actually exist. * </p> */ -public class EsFileSystem { +public class EsInstallation { private final File homeDirectory; private final List<File> outdatedSearchDirectories; private final File dataDirectory; private final File confDirectory; private final File logDirectory; - - public EsFileSystem(Props props) { + private EsJvmOptions esJvmOptions; + private EsYmlSettings esYmlSettings; + private Properties log4j2Properties; + private String clusterName; + private String host; + private int port; + + public EsInstallation(Props props) { File sqHomeDir = props.nonNullValueAsFile(ProcessProperties.PATH_HOME); this.homeDirectory = new File(sqHomeDir, "elasticsearch"); @@ -100,7 +108,7 @@ public class EsFileSystem { return "elasticsearch"; } - public File getLog4j2Properties() { + public File getLog4j2PropertiesLocation() { return new File(confDirectory, "log4j2.properties"); } @@ -111,4 +119,62 @@ public class EsFileSystem { public File getJvmOptions() { return new File(confDirectory, "jvm.options"); } + + public File getLibDirectory() { + return new File(homeDirectory, "lib"); + } + + public EsJvmOptions getEsJvmOptions() { + return esJvmOptions; + } + + public EsInstallation setEsJvmOptions(EsJvmOptions esJvmOptions) { + this.esJvmOptions = esJvmOptions; + return this; + } + + public EsYmlSettings getEsYmlSettings() { + return esYmlSettings; + } + + public EsInstallation setEsYmlSettings(EsYmlSettings esYmlSettings) { + this.esYmlSettings = esYmlSettings; + return this; + } + + public Properties getLog4j2Properties() { + return log4j2Properties; + } + + public EsInstallation setLog4j2Properties(Properties log4j2Properties) { + this.log4j2Properties = log4j2Properties; + return this; + } + + public String getClusterName() { + return clusterName; + } + + public EsInstallation setClusterName(String clusterName) { + this.clusterName = clusterName; + return this; + } + + public String getHost() { + return host; + } + + public EsInstallation setHost(String host) { + this.host = host; + return this; + } + + public int getPort() { + return port; + } + + public EsInstallation setPort(int port) { + this.port = port; + return this; + } } diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java index e8dc7e3b6eb..79e949163cd 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java @@ -42,13 +42,13 @@ public class EsSettings { private static final String STANDALONE_NODE_NAME = "sonarqube"; private final Props props; - private final EsFileSystem fileSystem; + private final EsInstallation fileSystem; private final boolean clusterEnabled; private final String clusterName; private final String nodeName; - public EsSettings(Props props, EsFileSystem fileSystem, System2 system2) { + public EsSettings(Props props, EsInstallation fileSystem, System2 system2) { this.props = props; this.fileSystem = fileSystem; diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/EsProcessMonitor.java b/server/sonar-main/src/main/java/org/sonar/application/process/EsProcessMonitor.java index f4b3ad9240d..3e3c13582ed 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/process/EsProcessMonitor.java +++ b/server/sonar-main/src/main/java/org/sonar/application/process/EsProcessMonitor.java @@ -38,7 +38,8 @@ import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.transport.Netty4Plugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.application.command.EsCommand; +import org.sonar.application.es.EsInstallation; +import org.sonar.process.ProcessId; import static java.util.Collections.singletonList; import static java.util.Collections.unmodifiableList; @@ -56,13 +57,13 @@ public class EsProcessMonitor extends AbstractProcessMonitor { private final AtomicBoolean nodeUp = new AtomicBoolean(false); private final AtomicBoolean nodeOperational = new AtomicBoolean(false); private final AtomicBoolean firstMasterNotDiscoveredLog = new AtomicBoolean(true); - private final EsCommand esCommand; + private final EsInstallation esConfig; private final EsConnector esConnector; private AtomicReference<TransportClient> transportClient = new AtomicReference<>(null); - public EsProcessMonitor(Process process, EsCommand esCommand, EsConnector esConnector) { - super(process, esCommand.getProcessId()); - this.esCommand = esCommand; + public EsProcessMonitor(Process process, ProcessId processId, EsInstallation esConfig, EsConnector esConnector) { + super(process, processId); + this.esConfig = esConfig; this.esConnector = esConnector; } @@ -169,10 +170,10 @@ public class EsProcessMonitor extends AbstractProcessMonitor { Settings.Builder esSettings = Settings.builder(); // mandatory property defined by bootstrap process - esSettings.put("cluster.name", esCommand.getClusterName()); + esSettings.put("cluster.name", esConfig.getClusterName()); TransportClient nativeClient = new MinimalTransportClient(esSettings.build()); - HostAndPort host = HostAndPort.fromParts(esCommand.getHost(), esCommand.getPort()); + HostAndPort host = HostAndPort.fromParts(esConfig.getHost(), esConfig.getPort()); addHostToClient(host, nativeClient); if (LOG.isDebugEnabled()) { LOG.debug("Connected to Elasticsearch node: [{}]", displayedAddresses(nativeClient)); diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncher.java b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncher.java index c0ca8ec7825..d420f368c92 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncher.java +++ b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncher.java @@ -20,8 +20,7 @@ package org.sonar.application.process; import java.io.Closeable; -import org.sonar.application.command.EsCommand; -import org.sonar.application.command.JavaCommand; +import org.sonar.application.command.AbstractCommand; public interface ProcessLauncher extends Closeable { @@ -29,16 +28,9 @@ public interface ProcessLauncher extends Closeable { void close(); /** - * Launch an ES command. + * Launch a command. * * @throws IllegalStateException if an error occurs */ - ProcessMonitor launch(EsCommand esCommand); - - /** - * Launch a Java command. - * - * @throws IllegalStateException if an error occurs - */ - ProcessMonitor launch(JavaCommand javaCommand); + ProcessMonitor launch(AbstractCommand command); } diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java index f067cc15c76..3da6b18a0b7 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java @@ -33,10 +33,10 @@ import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.application.command.AbstractCommand; -import org.sonar.application.command.EsCommand; +import org.sonar.application.command.EsScriptCommand; import org.sonar.application.command.JavaCommand; import org.sonar.application.command.JvmOptions; -import org.sonar.application.es.EsFileSystem; +import org.sonar.application.es.EsInstallation; import org.sonar.process.ProcessId; import org.sonar.process.sharedmemoryfile.AllProcessesCommands; import org.sonar.process.sharedmemoryfile.ProcessCommands; @@ -69,30 +69,51 @@ public class ProcessLauncherImpl implements ProcessLauncher { allProcessesCommands.close(); } - @Override - public ProcessMonitor launch(EsCommand esCommand) { - Process process = null; - try { - cleanupOutdatedEsData(esCommand); - writeConfFiles(esCommand); - ProcessBuilder processBuilder = create(esCommand); - logLaunchedCommand(esCommand, processBuilder); + public ProcessMonitor launch(AbstractCommand command) { + EsInstallation fileSystem = command.getEsInstallation(); + if (fileSystem != null) { + cleanupOutdatedEsData(fileSystem); + writeConfFiles(fileSystem); + } - process = processBuilder.start(); + Process process; + if (command instanceof EsScriptCommand) { + process = launchExternal((EsScriptCommand) command); + } else if (command instanceof JavaCommand) { + process = launchJava((JavaCommand) command); + } else { + throw new IllegalStateException("Unexpected type of command: " + command.getClass()); + } - return new EsProcessMonitor(process, esCommand, new EsConnectorImpl()); + ProcessId processId = command.getProcessId(); + try { + if (processId == ProcessId.ELASTICSEARCH) { + return new EsProcessMonitor(process, processId, command.getEsInstallation(), new EsConnectorImpl()); + } else { + ProcessCommands commands = allProcessesCommands.createAfterClean(processId.getIpcIndex()); + return new ProcessCommandsProcessMonitor(process, processId, commands); + } } catch (Exception e) { // just in case if (process != null) { process.destroyForcibly(); } - throw new IllegalStateException(format("Fail to launch process [%s]", esCommand.getProcessId().getKey()), e); + throw new IllegalStateException(format("Fail to launch monitor of process [%s]", processId.getKey()), e); } } - private static void cleanupOutdatedEsData(EsCommand esCommand) { - EsFileSystem esFileSystem = esCommand.getFileSystem(); - esFileSystem.getOutdatedSearchDirectories().forEach(outdatedDir -> { + private Process launchExternal(EsScriptCommand esScriptCommand) { + try { + ProcessBuilder processBuilder = create(esScriptCommand); + logLaunchedCommand(esScriptCommand, processBuilder); + return processBuilder.start(); + } catch (Exception e) { + throw new IllegalStateException(format("Fail to launch process [%s]", esScriptCommand.getProcessId().getKey()), e); + } + } + + private static void cleanupOutdatedEsData(EsInstallation esInstallation) { + esInstallation.getOutdatedSearchDirectories().forEach(outdatedDir -> { if (outdatedDir.exists()) { LOG.info("Deleting outdated search index data directory {}", outdatedDir.getAbsolutePath()); try { @@ -104,9 +125,8 @@ public class ProcessLauncherImpl implements ProcessLauncher { }); } - private static void writeConfFiles(EsCommand esCommand) { - EsFileSystem esFileSystem = esCommand.getFileSystem(); - File confDir = esFileSystem.getConfDirectory(); + private static void writeConfFiles(EsInstallation esInstallation) { + File confDir = esInstallation.getConfDirectory(); if (!confDir.exists() && !confDir.mkdirs()) { String error = format("Failed to create temporary configuration directory [%s]", confDir.getAbsolutePath()); LOG.error(error); @@ -114,30 +134,21 @@ public class ProcessLauncherImpl implements ProcessLauncher { } try { - esCommand.getEsYmlSettings().writeToYmlSettingsFile(esFileSystem.getElasticsearchYml()); - esCommand.getEsJvmOptions().writeToJvmOptionFile(esFileSystem.getJvmOptions()); - esCommand.getLog4j2Properties().store(new FileOutputStream(esFileSystem.getLog4j2Properties()), "log4j2 properties file for ES bundled in SonarQube"); + esInstallation.getEsYmlSettings().writeToYmlSettingsFile(esInstallation.getElasticsearchYml()); + esInstallation.getEsJvmOptions().writeToJvmOptionFile(esInstallation.getJvmOptions()); + esInstallation.getLog4j2Properties().store(new FileOutputStream(esInstallation.getLog4j2PropertiesLocation()), "log4j2 properties file for ES bundled in SonarQube"); } catch (IOException e) { throw new IllegalStateException("Failed to write ES configuration files", e); } } - @Override - public ProcessMonitor launch(JavaCommand javaCommand) { - Process process = null; + private Process launchJava(JavaCommand javaCommand) { ProcessId processId = javaCommand.getProcessId(); try { - ProcessCommands commands = allProcessesCommands.createAfterClean(processId.getIpcIndex()); - ProcessBuilder processBuilder = create(javaCommand); logLaunchedCommand(javaCommand, processBuilder); - process = processBuilder.start(); - return new ProcessCommandsProcessMonitor(process, processId, commands); + return processBuilder.start(); } catch (Exception e) { - // just in case - if (process != null) { - process.destroyForcibly(); - } throw new IllegalStateException(format("Fail to launch process [%s]", processId.getKey()), e); } } @@ -151,12 +162,12 @@ public class ProcessLauncherImpl implements ProcessLauncher { } } - private ProcessBuilder create(EsCommand esCommand) { + private ProcessBuilder create(EsScriptCommand esScriptCommand) { List<String> commands = new ArrayList<>(); - commands.add(esCommand.getFileSystem().getExecutable().getAbsolutePath()); - commands.addAll(esCommand.getEsOptions()); + commands.add(esScriptCommand.getEsInstallation().getExecutable().getAbsolutePath()); + commands.addAll(esScriptCommand.getOptions()); - return create(esCommand, commands); + return create(esScriptCommand, commands); } private <T extends JvmOptions> ProcessBuilder create(JavaCommand<T> javaCommand) { @@ -165,7 +176,15 @@ public class ProcessLauncherImpl implements ProcessLauncher { commands.addAll(javaCommand.getJvmOptions().getAll()); commands.addAll(buildClasspath(javaCommand)); commands.add(javaCommand.getClassName()); - commands.add(buildPropertiesFile(javaCommand).getAbsolutePath()); + if (javaCommand.getReadsArgumentsFromFile()) { + commands.add(buildPropertiesFile(javaCommand).getAbsolutePath()); + } else { + javaCommand.getArguments().forEach((key, value) -> { + if (value != null && !value.isEmpty()) { + commands.add("-E" + key + "=" + value); + } + }); + } return create(javaCommand, commands); } diff --git a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java index a3125ad6823..76fc63ea42a 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java @@ -40,7 +40,7 @@ import org.junit.rules.Timeout; import org.mockito.Mockito; import org.sonar.application.command.AbstractCommand; import org.sonar.application.command.CommandFactory; -import org.sonar.application.command.EsCommand; +import org.sonar.application.command.EsScriptCommand; import org.sonar.application.command.JavaCommand; import org.sonar.application.config.TestAppSettings; import org.sonar.application.process.ProcessLauncher; @@ -73,7 +73,8 @@ public class SchedulerImplTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - private EsCommand esCommand; + private EsScriptCommand esScriptCommand; + private JavaCommand esJavaCommand; private JavaCommand webLeaderCommand; private JavaCommand webFollowerCommand; private JavaCommand ceCommand; @@ -90,7 +91,8 @@ public class SchedulerImplTest { @Before public void setUp() throws Exception { File tempDir = temporaryFolder.newFolder(); - esCommand = new EsCommand(ELASTICSEARCH, tempDir); + esScriptCommand = new EsScriptCommand(ELASTICSEARCH, tempDir); + esJavaCommand = new JavaCommand(ELASTICSEARCH, tempDir); webLeaderCommand = new JavaCommand(WEB_SERVER, tempDir); webFollowerCommand = new JavaCommand(WEB_SERVER, tempDir); ceCommand = new JavaCommand(COMPUTE_ENGINE, tempDir); @@ -117,7 +119,7 @@ public class SchedulerImplTest { TestProcess web = processLauncher.waitForProcess(WEB_SERVER); assertThat(web.isAlive()).isTrue(); assertThat(processLauncher.processes).hasSize(2); - assertThat(processLauncher.commands).containsExactly(esCommand, webLeaderCommand); + assertThat(processLauncher.commands).containsExactly(esScriptCommand, webLeaderCommand); // web becomes operational -> CE is starting web.operational = true; @@ -125,7 +127,7 @@ public class SchedulerImplTest { TestProcess ce = processLauncher.waitForProcess(COMPUTE_ENGINE); assertThat(ce.isAlive()).isTrue(); assertThat(processLauncher.processes).hasSize(3); - assertThat(processLauncher.commands).containsExactly(esCommand, webLeaderCommand, ceCommand); + assertThat(processLauncher.commands).containsExactly(esScriptCommand, webLeaderCommand, ceCommand); // all processes are up processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isTrue()); @@ -360,8 +362,8 @@ public class SchedulerImplTest { private class TestCommandFactory implements CommandFactory { @Override - public EsCommand createEsCommand() { - return esCommand; + public EsScriptCommand createEsCommand() { + return esScriptCommand; } @Override @@ -381,13 +383,8 @@ public class SchedulerImplTest { private ProcessId makeStartupFail = null; @Override - public ProcessMonitor launch(EsCommand esCommand) { - return launchImpl(esCommand); - } - - @Override - public ProcessMonitor launch(JavaCommand javaCommand) { - return launchImpl(javaCommand); + public ProcessMonitor launch(AbstractCommand command) { + return launchImpl(command); } private ProcessMonitor launchImpl(AbstractCommand<?> javaCommand) { diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/AbstractCommandTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/AbstractCommandTest.java index c5ff76bc4dc..efecd933874 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/command/AbstractCommandTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/command/AbstractCommandTest.java @@ -67,28 +67,6 @@ public class AbstractCommandTest { } @Test - public void test_command_with_complete_information() throws Exception { - File workDir = temp.newFolder(); - AbstractCommand command = new AbstractCommand(ProcessId.ELASTICSEARCH, workDir, System2.INSTANCE) { - - }; - - command.setArgument("first_arg", "val1"); - Properties args = new Properties(); - args.setProperty("second_arg", "val2"); - command.setArguments(args); - - command.setEnvVariable("JAVA_COMMAND_TEST", "1000"); - - assertThat(command.toString()).isNotNull(); - assertThat(command.getWorkDir()).isSameAs(workDir); - - // copy current env variables - assertThat(command.getEnvVariables().get("JAVA_COMMAND_TEST")).isEqualTo("1000"); - assertThat(command.getEnvVariables().size()).isEqualTo(System.getenv().size() + 1); - } - - @Test public void setEnvVariable_fails_with_NPE_if_key_is_null() throws IOException { File workDir = temp.newFolder(); AbstractCommand underTest = new AbstractCommand(ProcessId.ELASTICSEARCH, workDir, System2.INSTANCE) { diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java index ceb13a08dac..32d29fd82eb 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java @@ -31,6 +31,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; +import org.sonar.application.es.EsInstallation; import org.sonar.process.ProcessId; import org.sonar.process.ProcessProperties; import org.sonar.process.Props; @@ -103,30 +104,66 @@ public class CommandFactoryImplTest { } @Test - public void createEsCommand_returns_command_for_default_settings() throws Exception { + public void createEsCommand_for_unix_returns_command_for_default_settings() throws Exception { + System2 system2 = Mockito.mock(System2.class); + when(system2.isOsWindows()).thenReturn(false); + prepareEsFileSystem(); + + Properties props = new Properties(); + props.setProperty("sonar.search.host", "localhost"); + + AbstractCommand esCommand = newFactory(props, system2).createEsCommand(); + EsInstallation esConfig = esCommand.getEsInstallation(); + + assertThat(esCommand).isInstanceOf(EsScriptCommand.class); + assertThat(esConfig.getClusterName()).isEqualTo("sonarqube"); + assertThat(esConfig.getHost()).isNotEmpty(); + assertThat(esConfig.getPort()).isEqualTo(9001); + assertThat(esConfig.getEsJvmOptions().getAll()) + // enforced values + .contains("-XX:+UseConcMarkSweepGC", "-server", "-Dfile.encoding=UTF-8") + // default settings + .contains("-Xms512m", "-Xmx512m", "-XX:+HeapDumpOnOutOfMemoryError"); + File esConfDir = new File(tempDir, "conf/es"); + assertThat(esCommand.getEnvVariables()) + .contains(entry("ES_JVM_OPTIONS", new File(esConfDir, "jvm.options").getAbsolutePath())) + .containsKey("JAVA_HOME"); + assertThat(esConfig.getEsYmlSettings()).isNotNull(); + + assertThat(esConfig.getLog4j2Properties()) + .contains(entry("appender.file_es.fileName", new File(logsDir, "es.log").getAbsolutePath())); + + assertThat(esCommand.getSuppressedEnvVariables()).containsOnly("JAVA_TOOL_OPTIONS"); + } + + @Test + public void createEsCommand_for_windows_returns_command_for_default_settings() throws Exception { + System2 system2 = Mockito.mock(System2.class); + when(system2.isOsWindows()).thenReturn(true); prepareEsFileSystem(); Properties props = new Properties(); props.setProperty("sonar.search.host", "localhost"); - EsCommand esCommand = newFactory(props).createEsCommand(); + AbstractCommand esCommand = newFactory(props, system2).createEsCommand(); + EsInstallation esConfig = esCommand.getEsInstallation(); - assertThat(esCommand.getClusterName()).isEqualTo("sonarqube"); - assertThat(esCommand.getHost()).isNotEmpty(); - assertThat(esCommand.getPort()).isEqualTo(9001); - assertThat(esCommand.getEsJvmOptions().getAll()) + assertThat(esCommand).isInstanceOf(JavaCommand.class); + assertThat(esConfig.getClusterName()).isEqualTo("sonarqube"); + assertThat(esConfig.getHost()).isNotEmpty(); + assertThat(esConfig.getPort()).isEqualTo(9001); + assertThat(esConfig.getEsJvmOptions().getAll()) // enforced values .contains("-XX:+UseConcMarkSweepGC", "-server", "-Dfile.encoding=UTF-8") // default settings .contains("-Xms512m", "-Xmx512m", "-XX:+HeapDumpOnOutOfMemoryError"); File esConfDir = new File(tempDir, "conf/es"); - assertThat(esCommand.getEsOptions()).containsOnly("-Epath.conf=" + esConfDir.getAbsolutePath()); assertThat(esCommand.getEnvVariables()) .contains(entry("ES_JVM_OPTIONS", new File(esConfDir, "jvm.options").getAbsolutePath())) .containsKey("JAVA_HOME"); - assertThat(esCommand.getEsYmlSettings()).isNotNull(); + assertThat(esConfig.getEsYmlSettings()).isNotNull(); - assertThat(esCommand.getLog4j2Properties()) + assertThat(esConfig.getLog4j2Properties()) .contains(entry("appender.file_es.fileName", new File(logsDir, "es.log").getAbsolutePath())); assertThat(esCommand.getSuppressedEnvVariables()).containsOnly("JAVA_TOOL_OPTIONS"); @@ -142,11 +179,12 @@ public class CommandFactoryImplTest { props.setProperty("sonar.search.port", "1234"); props.setProperty("sonar.search.javaOpts", "-Xms10G -Xmx10G"); - EsCommand command = newFactory(props).createEsCommand(); + AbstractCommand esCommand = newFactory(props).createEsCommand(); + EsInstallation esConfig = esCommand.getEsInstallation(); - assertThat(command.getClusterName()).isEqualTo("foo"); - assertThat(command.getPort()).isEqualTo(1234); - assertThat(command.getEsJvmOptions().getAll()) + assertThat(esConfig.getClusterName()).isEqualTo("foo"); + assertThat(esConfig.getPort()).isEqualTo(1234); + assertThat(esConfig.getEsJvmOptions().getAll()) // enforced values .contains("-XX:+UseConcMarkSweepGC", "-server", "-Dfile.encoding=UTF-8") // user settings @@ -171,7 +209,32 @@ public class CommandFactoryImplTest { .contains("-Xmx512m", "-Xms128m", "-XX:+HeapDumpOnOutOfMemoryError"); assertThat(command.getProcessId()).isEqualTo(ProcessId.WEB_SERVER); assertThat(command.getEnvVariables()) - .containsKey("JAVA_HOME"); + .isNotEmpty(); + assertThat(command.getArguments()) + // default settings + .contains(entry("sonar.web.javaOpts", "-Xmx512m -Xms128m -XX:+HeapDumpOnOutOfMemoryError")) + .contains(entry("sonar.cluster.enabled", "false")); + + assertThat(command.getSuppressedEnvVariables()).containsOnly("JAVA_TOOL_OPTIONS"); + } + + @Test + public void createCeCommand_returns_command_for_default_settings() throws Exception { + JavaCommand command = newFactory(new Properties()).createCeCommand(); + + assertThat(command.getClassName()).isEqualTo("org.sonar.ce.app.CeServer"); + assertThat(command.getWorkDir().getAbsolutePath()).isEqualTo(homeDir.getAbsolutePath()); + assertThat(command.getClasspath()) + .containsExactlyInAnyOrder("./lib/common/*", "./lib/server/*", "./lib/ce/*"); + assertThat(command.getJvmOptions().getAll()) + // enforced values + .contains("-Djava.awt.headless=true", "-Dfile.encoding=UTF-8") + // default settings + .contains("-Djava.io.tmpdir=" + tempDir.getAbsolutePath(), "-Dfile.encoding=UTF-8") + .contains("-Xmx512m", "-Xms128m", "-XX:+HeapDumpOnOutOfMemoryError"); + assertThat(command.getProcessId()).isEqualTo(ProcessId.COMPUTE_ENGINE); + assertThat(command.getEnvVariables()) + .isNotEmpty(); assertThat(command.getArguments()) // default settings .contains(entry("sonar.web.javaOpts", "-Xmx512m -Xms128m -XX:+HeapDumpOnOutOfMemoryError")) @@ -220,7 +283,11 @@ public class CommandFactoryImplTest { FileUtils.touch(new File(homeDir, "elasticsearch/bin/elasticsearch.bat")); } - private CommandFactory newFactory(Properties userProps) throws IOException { + private CommandFactoryImpl newFactory(Properties userProps) throws IOException { + return newFactory(userProps, System2.INSTANCE); + } + + private CommandFactoryImpl newFactory(Properties userProps, System2 system2) throws IOException { Properties p = new Properties(); p.setProperty("sonar.path.home", homeDir.getAbsolutePath()); p.setProperty("sonar.path.temp", tempDir.getAbsolutePath()); @@ -229,7 +296,7 @@ public class CommandFactoryImplTest { Props props = new Props(p); ProcessProperties.completeDefaults(props); - return new CommandFactoryImpl(props, tempDir, System2.INSTANCE); + return new CommandFactoryImpl(props, tempDir, system2); } private <T> void attachMemoryAppenderToLoggerOf(Class<T> loggerClass) { diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsFileSystemTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsInstallationTest.java index b2ae4c91cd1..3821d40f78c 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsFileSystemTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsInstallationTest.java @@ -31,7 +31,7 @@ import org.sonar.process.Props; import static org.assertj.core.api.Assertions.assertThat; -public class EsFileSystemTest { +public class EsInstallationTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); @@ -45,7 +45,7 @@ public class EsFileSystemTest { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Property sonar.path.home is not set"); - new EsFileSystem(props); + new EsInstallation(props); } @Test @@ -57,7 +57,7 @@ public class EsFileSystemTest { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Property sonar.path.temp is not set"); - new EsFileSystem(props); + new EsInstallation(props); } @Test @@ -68,7 +68,7 @@ public class EsFileSystemTest { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Missing property: sonar.path.data"); - new EsFileSystem(props); + new EsInstallation(props); } @Test @@ -80,7 +80,7 @@ public class EsFileSystemTest { props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath()); props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath()); - EsFileSystem underTest = new EsFileSystem(props); + EsInstallation underTest = new EsInstallation(props); assertThat(underTest.getHomeDirectory()).isEqualTo(new File(sqHomeDir, "elasticsearch")); } @@ -97,7 +97,7 @@ public class EsFileSystemTest { props.set(ProcessProperties.PATH_DATA, dataDir.getAbsolutePath()); - EsFileSystem underTest = new EsFileSystem(props); + EsInstallation underTest = new EsInstallation(props); assertThat(underTest.getDataDirectory()).isEqualTo(new File(dataDir, "es5")); } @@ -112,7 +112,7 @@ public class EsFileSystemTest { props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath()); props.set(ProcessProperties.PATH_LOGS, logDir.getAbsolutePath()); - EsFileSystem underTest = new EsFileSystem(props); + EsInstallation underTest = new EsInstallation(props); assertThat(underTest.getLogDirectory()).isEqualTo(logDir); } @@ -126,7 +126,7 @@ public class EsFileSystemTest { props.set(ProcessProperties.PATH_TEMP, tempDir.getAbsolutePath()); props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath()); - EsFileSystem underTest = new EsFileSystem(props); + EsInstallation underTest = new EsInstallation(props); assertThat(underTest.getConfDirectory()).isEqualTo(new File(tempDir, "conf/es")); } @@ -140,7 +140,7 @@ public class EsFileSystemTest { props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath()); props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath()); - EsFileSystem underTest = new EsFileSystem(props); + EsInstallation underTest = new EsInstallation(props); if (System.getProperty("os.name").startsWith("Windows")) { assertThat(underTest.getExecutable()).isEqualTo(new File(sqHomeDir, "elasticsearch/bin/elasticsearch.bat")); @@ -158,9 +158,9 @@ public class EsFileSystemTest { props.set(ProcessProperties.PATH_TEMP, tempDir.getAbsolutePath()); props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath()); - EsFileSystem underTest = new EsFileSystem(props); + EsInstallation underTest = new EsInstallation(props); - assertThat(underTest.getLog4j2Properties()).isEqualTo(new File(tempDir, "conf/es/log4j2.properties")); + assertThat(underTest.getLog4j2PropertiesLocation()).isEqualTo(new File(tempDir, "conf/es/log4j2.properties")); } @Test @@ -172,7 +172,7 @@ public class EsFileSystemTest { props.set(ProcessProperties.PATH_TEMP, tempDir.getAbsolutePath()); props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath()); - EsFileSystem underTest = new EsFileSystem(props); + EsInstallation underTest = new EsInstallation(props); assertThat(underTest.getElasticsearchYml()).isEqualTo(new File(tempDir, "conf/es/elasticsearch.yml")); } @@ -186,7 +186,7 @@ public class EsFileSystemTest { props.set(ProcessProperties.PATH_TEMP, tempDir.getAbsolutePath()); props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath()); - EsFileSystem underTest = new EsFileSystem(props); + EsInstallation underTest = new EsInstallation(props); assertThat(underTest.getJvmOptions()).isEqualTo(new File(tempDir, "conf/es/jvm.options")); } diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java index ff6db104e8b..3cc7580e2be 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java @@ -65,7 +65,7 @@ public class EsSettingsTest { this.listAppender = ListAppender.attachMemoryAppenderToLoggerOf(EsSettings.class); Props props = minimalProps(); System2 system2 = mock(System2.class); - new EsSettings(props, new EsFileSystem(props), system2); + new EsSettings(props, new EsInstallation(props), system2); assertThat(listAppender.getLogs()).isEmpty(); } @@ -76,7 +76,7 @@ public class EsSettingsTest { Props props = minimalProps(); System2 system2 = mock(System2.class); when(system2.getenv("ES_JVM_OPTIONS")).thenReturn(" "); - new EsSettings(props, new EsFileSystem(props), system2); + new EsSettings(props, new EsInstallation(props), system2); assertThat(listAppender.getLogs()).isEmpty(); } @@ -87,7 +87,7 @@ public class EsSettingsTest { Props props = minimalProps(); System2 system2 = mock(System2.class); when(system2.getenv("ES_JVM_OPTIONS")).thenReturn(randomAlphanumeric(2)); - new EsSettings(props, new EsFileSystem(props), system2); + new EsSettings(props, new EsInstallation(props), system2); assertThat(listAppender.getLogs()) .extracting(ILoggingEvent::getMessage) @@ -117,7 +117,7 @@ public class EsSettingsTest { props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath()); props.set(CLUSTER_NAME, "sonarqube"); - EsSettings esSettings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE); + EsSettings esSettings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); Map<String, String> generated = esSettings.build(); assertThat(generated.get("transport.tcp.port")).isEqualTo("1234"); @@ -156,7 +156,7 @@ public class EsSettingsTest { props.set(ProcessProperties.CLUSTER_ENABLED, "true"); props.set(ProcessProperties.CLUSTER_NODE_NAME, "node-1"); - EsSettings esSettings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE); + EsSettings esSettings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); Map<String, String> generated = esSettings.build(); assertThat(generated.get("cluster.name")).isEqualTo("sonarqube-1"); @@ -175,7 +175,7 @@ public class EsSettingsTest { props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath()); props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath()); props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath()); - EsSettings esSettings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE); + EsSettings esSettings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); Map<String, String> generated = esSettings.build(); assertThat(generated.get("node.name")).startsWith("sonarqube-"); } @@ -192,20 +192,20 @@ public class EsSettingsTest { props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath()); props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath()); props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath()); - EsSettings esSettings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE); + EsSettings esSettings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); Map<String, String> generated = esSettings.build(); assertThat(generated.get("node.name")).isEqualTo("sonarqube"); } @Test public void path_properties_are_values_from_EsFileSystem_argument() throws IOException { - EsFileSystem mockedEsFileSystem = mock(EsFileSystem.class); - when(mockedEsFileSystem.getHomeDirectory()).thenReturn(new File("/foo/home")); - when(mockedEsFileSystem.getConfDirectory()).thenReturn(new File("/foo/conf")); - when(mockedEsFileSystem.getLogDirectory()).thenReturn(new File("/foo/log")); - when(mockedEsFileSystem.getDataDirectory()).thenReturn(new File("/foo/data")); + EsInstallation mockedEsInstallation = mock(EsInstallation.class); + when(mockedEsInstallation.getHomeDirectory()).thenReturn(new File("/foo/home")); + when(mockedEsInstallation.getConfDirectory()).thenReturn(new File("/foo/conf")); + when(mockedEsInstallation.getLogDirectory()).thenReturn(new File("/foo/log")); + when(mockedEsInstallation.getDataDirectory()).thenReturn(new File("/foo/data")); - EsSettings underTest = new EsSettings(minProps(new Random().nextBoolean()), mockedEsFileSystem, System2.INSTANCE); + EsSettings underTest = new EsSettings(minProps(new Random().nextBoolean()), mockedEsInstallation, System2.INSTANCE); Map<String, String> generated = underTest.build(); assertThat(generated.get("path.data")).isEqualTo("/foo/data"); @@ -217,7 +217,7 @@ public class EsSettingsTest { public void set_discovery_settings_if_cluster_is_enabled() throws Exception { Props props = minProps(CLUSTER_ENABLED); props.set(CLUSTER_SEARCH_HOSTS, "1.2.3.4:9000,1.2.3.5:8080"); - Map<String, String> settings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE).build(); + Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); assertThat(settings.get("discovery.zen.ping.unicast.hosts")).isEqualTo("1.2.3.4:9000,1.2.3.5:8080"); assertThat(settings.get("discovery.zen.minimum_master_nodes")).isEqualTo("2"); @@ -229,7 +229,7 @@ public class EsSettingsTest { Props props = minProps(CLUSTER_ENABLED); props.set(ProcessProperties.SEARCH_MINIMUM_MASTER_NODES, "ꝱꝲꝳପ"); - EsSettings underTest = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE); + EsSettings underTest = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("Value of property sonar.search.minimumMasterNodes is not an integer:"); @@ -240,7 +240,7 @@ public class EsSettingsTest { public void cluster_is_enabled_with_defined_minimum_master_nodes() throws Exception { Props props = minProps(CLUSTER_ENABLED); props.set(ProcessProperties.SEARCH_MINIMUM_MASTER_NODES, "5"); - Map<String, String> settings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE).build(); + Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); assertThat(settings.get("discovery.zen.minimum_master_nodes")).isEqualTo("5"); } @@ -249,7 +249,7 @@ public class EsSettingsTest { public void cluster_is_enabled_with_defined_initialTimeout() throws Exception { Props props = minProps(CLUSTER_ENABLED); props.set(ProcessProperties.SEARCH_INITIAL_STATE_TIMEOUT, "10s"); - Map<String, String> settings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE).build(); + Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); assertThat(settings.get("discovery.initial_state_timeout")).isEqualTo("10s"); } @@ -258,7 +258,7 @@ public class EsSettingsTest { public void in_standalone_initialTimeout_is_not_overridable() throws Exception { Props props = minProps(CLUSTER_DISABLED); props.set(ProcessProperties.SEARCH_INITIAL_STATE_TIMEOUT, "10s"); - Map<String, String> settings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE).build(); + Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); assertThat(settings.get("discovery.initial_state_timeout")).isEqualTo("30s"); } @@ -267,7 +267,7 @@ public class EsSettingsTest { public void in_standalone_minimumMasterNodes_is_not_overridable() throws Exception { Props props = minProps(CLUSTER_DISABLED); props.set(ProcessProperties.SEARCH_MINIMUM_MASTER_NODES, "5"); - Map<String, String> settings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE).build(); + Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); assertThat(settings.get("discovery.zen.minimum_master_nodes")).isEqualTo("1"); } @@ -276,7 +276,7 @@ public class EsSettingsTest { public void enable_http_connector() throws Exception { Props props = minProps(CLUSTER_DISABLED); props.set(ProcessProperties.SEARCH_HTTP_PORT, "9010"); - Map<String, String> settings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE).build(); + Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); assertThat(settings.get("http.port")).isEqualTo("9010"); assertThat(settings.get("http.host")).isEqualTo("127.0.0.1"); @@ -288,7 +288,7 @@ public class EsSettingsTest { Props props = minProps(CLUSTER_DISABLED); props.set(ProcessProperties.SEARCH_HTTP_PORT, "9010"); props.set(ProcessProperties.SEARCH_HOST, "127.0.0.2"); - Map<String, String> settings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE).build(); + Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); assertThat(settings.get("http.port")).isEqualTo("9010"); assertThat(settings.get("http.host")).isEqualTo("127.0.0.2"); diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/EsProcessMonitorTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/EsProcessMonitorTest.java index 7af20a08cc1..3c80c1c2429 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/process/EsProcessMonitorTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/process/EsProcessMonitorTest.java @@ -28,14 +28,16 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Properties; import java.util.Random; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.junit.Test; import org.slf4j.LoggerFactory; +import org.sonar.application.es.EsInstallation; import org.sonar.process.ProcessId; -import org.sonar.application.command.EsCommand; +import org.sonar.process.Props; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; @@ -49,7 +51,7 @@ public class EsProcessMonitorTest { public void isOperational_should_return_false_if_Elasticsearch_is_RED() throws Exception { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus(any())).thenReturn(ClusterHealthStatus.RED); - EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), getEsCommand(), esConnector); + EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), ProcessId.ELASTICSEARCH, getEsConfig(), esConnector); assertThat(underTest.isOperational()).isFalse(); } @@ -57,7 +59,7 @@ public class EsProcessMonitorTest { public void isOperational_should_return_true_if_Elasticsearch_is_YELLOW() throws Exception { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus(any())).thenReturn(ClusterHealthStatus.YELLOW); - EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), getEsCommand(), esConnector); + EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), ProcessId.ELASTICSEARCH, getEsConfig(), esConnector); assertThat(underTest.isOperational()).isTrue(); } @@ -65,7 +67,7 @@ public class EsProcessMonitorTest { public void isOperational_should_return_true_if_Elasticsearch_is_GREEN() throws Exception { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus(any())).thenReturn(ClusterHealthStatus.GREEN); - EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), getEsCommand(), esConnector); + EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), ProcessId.ELASTICSEARCH, getEsConfig(), esConnector); assertThat(underTest.isOperational()).isTrue(); } @@ -73,7 +75,7 @@ public class EsProcessMonitorTest { public void isOperational_should_return_true_if_Elasticsearch_was_GREEN_once() throws Exception { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus(any())).thenReturn(ClusterHealthStatus.GREEN); - EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), getEsCommand(), esConnector); + EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), ProcessId.ELASTICSEARCH, getEsConfig(), esConnector); assertThat(underTest.isOperational()).isTrue(); when(esConnector.getClusterHealthStatus(any())).thenReturn(ClusterHealthStatus.RED); @@ -86,7 +88,7 @@ public class EsProcessMonitorTest { when(esConnector.getClusterHealthStatus(any())) .thenThrow(new NoNodeAvailableException("test")) .thenReturn(ClusterHealthStatus.GREEN); - EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), getEsCommand(), esConnector); + EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), ProcessId.ELASTICSEARCH, getEsConfig(), esConnector); assertThat(underTest.isOperational()).isTrue(); } @@ -95,7 +97,7 @@ public class EsProcessMonitorTest { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus(any())) .thenThrow(new RuntimeException("test")); - EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), getEsCommand(), esConnector); + EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), ProcessId.ELASTICSEARCH, getEsConfig(), esConnector); assertThat(underTest.isOperational()).isFalse(); } @@ -112,7 +114,7 @@ public class EsProcessMonitorTest { when(esConnector.getClusterHealthStatus(any())) .thenThrow(new MasterNotDiscoveredException("Master not elected -test-")); - EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), getEsCommand(), esConnector); + EsProcessMonitor underTest = new EsProcessMonitor(mock(Process.class), ProcessId.ELASTICSEARCH, getEsConfig(), esConnector); assertThat(underTest.isOperational()).isFalse(); assertThat(memoryAppender.events).isNotEmpty(); assertThat(memoryAppender.events) @@ -130,9 +132,14 @@ public class EsProcessMonitorTest { ); } - private EsCommand getEsCommand() throws IOException { + private EsInstallation getEsConfig() throws IOException { Path tempDirectory = Files.createTempDirectory(getClass().getSimpleName()); - return new EsCommand(ProcessId.ELASTICSEARCH, tempDirectory.toFile()) + Properties properties = new Properties(); + properties.setProperty("sonar.path.home", "/imaginary/path"); + properties.setProperty("sonar.path.data", "/imaginary/path"); + properties.setProperty("sonar.path.temp", "/imaginary/path"); + properties.setProperty("sonar.path.logs", "/imaginary/path"); + return new EsInstallation(new Props(properties)) .setHost("localhost") .setPort(new Random().nextInt(40000)); } diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/ProcessLauncherImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/ProcessLauncherImplTest.java index d98723eb892..2f6a39edfc1 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/process/ProcessLauncherImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/process/ProcessLauncherImplTest.java @@ -30,11 +30,11 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; -import org.sonar.application.command.EsCommand; import org.sonar.application.command.EsJvmOptions; +import org.sonar.application.command.EsScriptCommand; import org.sonar.application.command.JavaCommand; import org.sonar.application.command.JvmOptions; -import org.sonar.application.es.EsFileSystem; +import org.sonar.application.es.EsInstallation; import org.sonar.application.es.EsYmlSettings; import org.sonar.process.ProcessId; import org.sonar.process.Props; @@ -93,7 +93,8 @@ public class ProcessLauncherImplTest { File tempDir = temp.newFolder(); TestProcessBuilder processBuilder = new TestProcessBuilder(); ProcessLauncher underTest = new ProcessLauncherImpl(tempDir, commands, () -> processBuilder); - JavaCommand<JvmOptions> command = new JavaCommand<>(ProcessId.ELASTICSEARCH, temp.newFolder()); + JavaCommand<JvmOptions> command = new JavaCommand<>(ProcessId.WEB_SERVER, temp.newFolder()); + command.setReadsArgumentsFromFile(true); command.setArgument("foo", "bar"); command.setArgument("baz", "woo"); command.setJvmOptions(new JvmOptions<>()); @@ -110,20 +111,38 @@ public class ProcessLauncherImplTest { entry("foo", "bar"), entry("baz", "woo"), entry("process.terminationTimeout", "60000"), - entry("process.key", ProcessId.ELASTICSEARCH.getKey()), - entry("process.index", String.valueOf(ProcessId.ELASTICSEARCH.getIpcIndex())), + entry("process.key", ProcessId.WEB_SERVER.getKey()), + entry("process.index", String.valueOf(ProcessId.WEB_SERVER.getIpcIndex())), entry("process.sharedDir", tempDir.getAbsolutePath())); } } @Test + public void temporary_properties_file_can_be_avoided() throws Exception { + File tempDir = temp.newFolder(); + TestProcessBuilder processBuilder = new TestProcessBuilder(); + ProcessLauncher underTest = new ProcessLauncherImpl(tempDir, commands, () -> processBuilder); + JavaCommand<JvmOptions> command = new JavaCommand<>(ProcessId.WEB_SERVER, temp.newFolder()); + command.setReadsArgumentsFromFile(false); + command.setArgument("foo", "bar"); + command.setArgument("baz", "woo"); + command.setJvmOptions(new JvmOptions<>()); + + underTest.launch(command); + + String propsFilePath = processBuilder.commands.get(processBuilder.commands.size() - 1); + File file = new File(propsFilePath); + assertThat(file).doesNotExist(); + } + + @Test public void clean_up_old_es_data() throws Exception { File tempDir = temp.newFolder(); File homeDir = temp.newFolder(); File dataDir = temp.newFolder(); File logDir = temp.newFolder(); ProcessLauncher underTest = new ProcessLauncherImpl(tempDir, commands, () -> new TestProcessBuilder()); - EsCommand command = createEsCommand(tempDir, homeDir, dataDir, logDir); + EsScriptCommand command = createEsScriptCommand(tempDir, homeDir, dataDir, logDir); File outdatedEsDir = new File(dataDir, "es"); assertThat(outdatedEsDir.mkdir()).isTrue(); @@ -141,7 +160,7 @@ public class ProcessLauncherImplTest { File dataDir = temp.newFolder(); File logDir = temp.newFolder(); ProcessLauncher underTest = new ProcessLauncherImpl(tempDir, commands, () -> new TestProcessBuilder()); - EsCommand command = createEsCommand(tempDir, homeDir, dataDir, logDir); + EsScriptCommand command = createEsScriptCommand(tempDir, homeDir, dataDir, logDir); File outdatedEsDir = new File(dataDir, "es"); assertThat(outdatedEsDir.exists()).isFalse(); @@ -164,17 +183,17 @@ public class ProcessLauncherImplTest { underTest.launch(new JavaCommand(ProcessId.ELASTICSEARCH, temp.newFolder())); } - private EsCommand createEsCommand(File tempDir, File homeDir, File dataDir, File logDir) throws IOException { - EsCommand command = new EsCommand(ProcessId.ELASTICSEARCH, temp.newFolder()); + private EsScriptCommand createEsScriptCommand(File tempDir, File homeDir, File dataDir, File logDir) throws IOException { + EsScriptCommand command = new EsScriptCommand(ProcessId.ELASTICSEARCH, temp.newFolder()); Props props = new Props(new Properties()); props.set("sonar.path.temp", tempDir.getAbsolutePath()); props.set("sonar.path.home", homeDir.getAbsolutePath()); props.set("sonar.path.data", dataDir.getAbsolutePath()); props.set("sonar.path.logs", logDir.getAbsolutePath()); - command.setFileSystem(new EsFileSystem(props)); - command.setEsYmlSettings(mock(EsYmlSettings.class)); - command.setEsJvmOptions(mock(EsJvmOptions.class)); - command.setLog4j2Properties(new Properties()); + command.setEsInstallation(new EsInstallation(props) + .setEsYmlSettings(mock(EsYmlSettings.class)) + .setEsJvmOptions(mock(EsJvmOptions.class)) + .setLog4j2Properties(new Properties())); return command; } diff --git a/server/sonar-process/src/main/java/org/sonar/process/System2.java b/server/sonar-process/src/main/java/org/sonar/process/System2.java index 1c9ca5c51c6..0e32165046c 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/System2.java +++ b/server/sonar-process/src/main/java/org/sonar/process/System2.java @@ -20,6 +20,7 @@ package org.sonar.process; import java.util.Map; +import org.apache.commons.lang.SystemUtils; /** * An interface allowing to wrap around static call to {@link System} class. @@ -35,6 +36,11 @@ public interface System2 { public String getenv(String name) { return System.getenv(name); } + + @Override + public boolean isOsWindows() { + return SystemUtils.IS_OS_WINDOWS; + } }; /** @@ -46,4 +52,9 @@ public interface System2 { * Proxy to {@link System#getenv(String)}. */ String getenv(String name); + + /** + * True if this is MS Windows. + */ + boolean isOsWindows(); } diff --git a/server/sonar-server/src/main/java/org/sonar/ce/http/CeHttpClientImpl.java b/server/sonar-server/src/main/java/org/sonar/ce/http/CeHttpClientImpl.java index e119aa7948b..9f354316975 100644 --- a/server/sonar-server/src/main/java/org/sonar/ce/http/CeHttpClientImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/ce/http/CeHttpClientImpl.java @@ -51,6 +51,7 @@ public class CeHttpClientImpl implements CeHttpClient { /** * Connects to the specified JVM process and requests system information. + * * @return the system info, or absent if the process is not up or if its HTTP URL * is not registered into IPC. */ @@ -108,16 +109,17 @@ public class CeHttpClientImpl implements CeHttpClient { .post(RequestBody.create(null, new byte[0])) .url(url + "?level=" + newLogLevel.name()) .build(); - okhttp3.Response response = new OkHttpClient().newCall(request).execute(); - if (response.code() != 200) { - throw new IOException( - String.format( - "Failed to change log level in Compute Engine. Code was '%s' and response was '%s' for url '%s'", - response.code(), - response.body().string(), - url)); + try (okhttp3.Response response = new OkHttpClient().newCall(request).execute()) { + if (response.code() != 200) { + throw new IOException( + String.format( + "Failed to change log level in Compute Engine. Code was '%s' and response was '%s' for url '%s'", + response.code(), + response.body().string(), + url)); + } + return null; } - return null; } } @@ -145,16 +147,17 @@ public class CeHttpClientImpl implements CeHttpClient { .post(RequestBody.create(null, new byte[0])) .url(url) .build(); - okhttp3.Response response = new OkHttpClient().newCall(request).execute(); - if (response.code() != 200) { - throw new IOException( - String.format( - "Failed to trigger refresh of CE Worker count. Code was '%s' and response was '%s' for url '%s'", - response.code(), - response.body().string(), - url)); + try (okhttp3.Response response = new OkHttpClient().newCall(request).execute()) { + if (response.code() != 200) { + throw new IOException( + String.format( + "Failed to trigger refresh of CE Worker count. Code was '%s' and response was '%s' for url '%s'", + response.code(), + response.body().string(), + url)); + } + return null; } - return null; } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/issue/IssueCreationDateCalculator.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/issue/IssueCreationDateCalculator.java index 1ed3b618c8a..b8075217f9e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/issue/IssueCreationDateCalculator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/issue/IssueCreationDateCalculator.java @@ -23,6 +23,7 @@ import java.time.format.DateTimeFormatter; import java.util.Comparator; import java.util.Date; import java.util.HashSet; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; @@ -94,7 +95,7 @@ public class IssueCreationDateCalculator extends IssueVisitor { if (pluginKey == null) { return false; } - + ScannerPlugin scannerPlugin = Optional.ofNullable(analysisMetadataHolder.getScannerPluginsByKey().get(pluginKey)) .orElseThrow(illegalStateException("The rule %s is declared to come from plugin %s, but this plugin was not used by scanner.", activeRule.getRuleKey(), pluginKey)); return pluginIsNew(scannerPlugin, lastAnalysisDate) @@ -121,7 +122,7 @@ public class IssueCreationDateCalculator extends IssueVisitor { private Optional<Date> getScmChangeDate(Component component, DefaultIssue issue) { return getScmInfo(component) - .flatMap(scmInfo -> getChangeset(scmInfo, issue)) + .flatMap(scmInfo -> getChangeset(component, scmInfo, issue)) .map(IssueCreationDateCalculator::getChangeDate); } @@ -133,7 +134,7 @@ public class IssueCreationDateCalculator extends IssueVisitor { return toJavaUtilOptional(scmInfoRepository.getScmInfo(component)); } - private static Optional<Changeset> getChangeset(ScmInfo scmInfo, DefaultIssue issue) { + private static Optional<Changeset> getChangeset(Component component, ScmInfo scmInfo, DefaultIssue issue) { Set<Integer> involvedLines = new HashSet<>(); DbIssues.Locations locations = issue.getLocations(); if (locations != null) { @@ -142,7 +143,10 @@ public class IssueCreationDateCalculator extends IssueVisitor { } for (Flow f : locations.getFlowList()) { for (Location l : f.getLocationList()) { - addLines(involvedLines, l.getTextRange()); + if (Objects.equals(l.getComponentId(), component.getUuid())) { + // Ignore locations in other files, since it is currently not very common, and this is hard to load SCM by component UUID. + addLines(involvedLines, l.getTextRange()); + } } } if (!involvedLines.isEmpty()) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImpl.java index abab82d92eb..d6f1a394c09 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImpl.java @@ -45,7 +45,7 @@ public class ScmInfoRepositoryImpl implements ScmInfoRepository { @Override public Optional<ScmInfo> getScmInfo(Component component) { - requireNonNull(component, "Component cannot be bull"); + requireNonNull(component, "Component cannot be null"); return initializeScmInfoForComponent(component); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryClient.java b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryClient.java index 51f5e86f461..25c0a74f9c0 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/telemetry/TelemetryClient.java @@ -21,10 +21,12 @@ package org.sonar.server.telemetry; import java.io.IOException; +import okhttp3.Call; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; +import okhttp3.Response; import org.sonar.api.config.Configuration; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.log.Logger; @@ -47,7 +49,7 @@ public class TelemetryClient { void upload(String json) throws IOException { Request request = buildHttpRequest(json); - okHttpClient.newCall(request).execute(); + execute(okHttpClient.newCall(request)); } void optOut(String json) { @@ -57,7 +59,7 @@ public class TelemetryClient { request.delete(body); try { - okHttpClient.newCall(request.build()).execute(); + execute(okHttpClient.newCall(request.build())); } catch (IOException e) { LOG.debug("Error when sending opt-out usage statistics: {}", e.getMessage()); } @@ -75,4 +77,10 @@ public class TelemetryClient { return config.get(PROP_URL).orElseThrow(() -> new IllegalStateException(String.format("Setting '%s' must be provided.", PROP_URL))); } + private static void execute(Call call) throws IOException { + try (Response ignored = call.execute()) { + // auto close connection to avoid leaked connection + } + } + } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/issue/IssueCreationDateCalculatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/issue/IssueCreationDateCalculatorTest.java index 8f3cf22439e..3555c069dcf 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/issue/IssueCreationDateCalculatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/issue/IssueCreationDateCalculatorTest.java @@ -54,6 +54,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class IssueCreationDateCalculatorTest { + private static final String COMPONENT_UUID = "ab12"; + @Rule public AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule(); @@ -78,6 +80,7 @@ public class IssueCreationDateCalculatorTest { issueUpdater = mock(IssueFieldsSetter.class); activeRulesHolder = mock(ActiveRulesHolder.class); component = mock(Component.class); + when(component.getUuid()).thenReturn(COMPONENT_UUID); ruleKey = RuleKey.of("reop", "rule"); issue = mock(DefaultIssue.class); activeRule = mock(ActiveRule.class); @@ -253,8 +256,8 @@ public class IssueCreationDateCalculatorTest { Flow.Builder secondary = Flow.newBuilder().addLocation(Location.newBuilder().setTextRange(range(4, 5))); builder.addFlow(secondary).build(); Flow.Builder flow = Flow.newBuilder() - .addLocation(Location.newBuilder().setTextRange(range(6, 7))) - .addLocation(Location.newBuilder().setTextRange(range(8, 9))); + .addLocation(Location.newBuilder().setTextRange(range(6, 7)).setComponentId(COMPONENT_UUID)) + .addLocation(Location.newBuilder().setTextRange(range(8, 9)).setComponentId(COMPONENT_UUID)); builder.addFlow(flow).build(); when(issue.getLocations()).thenReturn(builder.build()); withScmAt(2, 1200L); @@ -271,6 +274,35 @@ public class IssueCreationDateCalculatorTest { assertChangeOfCreationDateTo(1900L); } + @Test + public void should_ignore_flows_location_outside_current_file_when_backdating() { + analysisMetadataHolder.setBaseAnalysis(null); + currentAnalysisIs(3000L); + + newIssue(); + Builder builder = DbIssues.Locations.newBuilder() + .setTextRange(range(2, 3)); + Flow.Builder secondary = Flow.newBuilder().addLocation(Location.newBuilder().setTextRange(range(4, 5))); + builder.addFlow(secondary).build(); + Flow.Builder flow = Flow.newBuilder() + .addLocation(Location.newBuilder().setTextRange(range(6, 7)).setComponentId(COMPONENT_UUID)) + .addLocation(Location.newBuilder().setTextRange(range(8, 9)).setComponentId("another")); + builder.addFlow(flow).build(); + when(issue.getLocations()).thenReturn(builder.build()); + withScmAt(2, 1200L); + withScmAt(3, 1300L); + withScmAt(4, 1400L); + withScmAt(5, 1500L); + withScmAt(6, 1600L); + withScmAt(7, 1700L); + withScmAt(8, 1800L); + withScmAt(9, 1900L); + + run(); + + assertChangeOfCreationDateTo(1700L); + } + private org.sonar.db.protobuf.DbCommons.TextRange.Builder range(int startLine, int endLine) { return TextRange.newBuilder().setStartLine(startLine).setEndLine(endLine); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java index d11354b7a12..6005811b7be 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java @@ -120,7 +120,7 @@ public class ScmInfoRepositoryImplTest { @Test public void fail_with_NPE_when_component_is_null() throws Exception { thrown.expect(NullPointerException.class); - thrown.expectMessage("Component cannot be bull"); + thrown.expectMessage("Component cannot be null"); underTest.getScmInfo(null); } diff --git a/server/sonar-web/src/main/js/app/components/nav/global/GlobalNav.css b/server/sonar-web/src/main/js/app/components/nav/global/GlobalNav.css index 3bbf08445ed..66c1a5daf30 100644 --- a/server/sonar-web/src/main/js/app/components/nav/global/GlobalNav.css +++ b/server/sonar-web/src/main/js/app/components/nav/global/GlobalNav.css @@ -19,6 +19,7 @@ .navbar-login { margin-right: -10px; + white-space: nowrap; } .navbar-avatar { diff --git a/server/sonar-web/src/main/js/components/common/ScreenPositionHelper.tsx b/server/sonar-web/src/main/js/components/common/ScreenPositionHelper.tsx index a3e9d99d0e2..5f7b2476f5b 100644 --- a/server/sonar-web/src/main/js/components/common/ScreenPositionHelper.tsx +++ b/server/sonar-web/src/main/js/components/common/ScreenPositionHelper.tsx @@ -48,7 +48,10 @@ export default class ScreenPositionHelper extends React.PureComponent<Props> { if (!containerPos) { return { top: 0, left: 0 }; } - return { top: window.scrollY + containerPos.top, left: window.scrollX + containerPos.left }; + return { + top: window.pageYOffset + containerPos.top, + left: window.pageXOffset + containerPos.left + }; }; render() { diff --git a/travis.sh b/travis.sh index f6f5977cc52..fb8ace1f6d3 100755 --- a/travis.sh +++ b/travis.sh @@ -163,7 +163,11 @@ BUILD) -Dsonar.host.url=$SONAR_HOST_URL \ -Dsonar.login=$SONAR_TOKEN \ -Dsonar.branch.name=$TRAVIS_BRANCH \ - -Dsonar.projectVersion=$INITIAL_VERSION + -Dsonar.projectVersion=$INITIAL_VERSION \ + -Dsonar.analysis.buildNumber=$TRAVIS_BUILD_NUMBER \ + -Dsonar.analysis.pipeline=$TRAVIS_BUILD_NUMBER \ + -Dsonar.analysis.sha1=$TRAVIS_COMMIT \ + -Dsonar.analysis.repository=$TRAVIS_REPO_SLUG elif [ "$TRAVIS_PULL_REQUEST" != "false" ] && [ -n "${GITHUB_TOKEN:-}" ]; then echo 'Build and analyze internal pull request' @@ -190,7 +194,11 @@ BUILD) -Dsonar.host.url=$SONAR_HOST_URL \ -Dsonar.login=$SONAR_TOKEN \ -Dsonar.branch.name=$TRAVIS_PULL_REQUEST_BRANCH \ - -Dsonar.branch.target=$TRAVIS_BRANCH + -Dsonar.branch.target=$TRAVIS_BRANCH \ + -Dsonar.analysis.buildNumber=$TRAVIS_BUILD_NUMBER \ + -Dsonar.analysis.pipeline=$TRAVIS_BUILD_NUMBER \ + -Dsonar.analysis.sha1=$TRAVIS_COMMIT \ + -Dsonar.analysis.repository=$TRAVIS_REPO_SLUG fi else echo 'Build feature branch or external pull request' |