From b1e40415a161a5ecdad7a321ac22a1b30710301f Mon Sep 17 00:00:00 2001 From: Michal Duda Date: Wed, 12 Feb 2020 15:54:09 +0100 Subject: [PATCH] SONAR-13078 enable configuring the server by environment variables --- .../config/AppSettingsLoaderImpl.java | 30 +++++++- .../config/AppSettingsLoaderImplTest.java | 74 ++++++++++++++++--- .../sonar/application/es/EsSettingsTest.java | 47 ++++++------ .../org/sonar/process/ProcessProperties.java | 5 ++ .../server/setting/ThreadLocalSettings.java | 59 +++++++++------ .../setting/ThreadLocalSettingsTest.java | 47 ++++++------ .../src/main/assembly/conf/sonar.properties | 4 +- .../main/java/org/sonar/application/App.java | 3 +- .../core/config/CorePropertyDefinitions.java | 7 ++ .../org/sonar/core/util/SettingFormatter.java | 32 ++++++++ .../config/CorePropertyDefinitionsTest.java | 2 +- .../sonar/core/util/SettingFormatterTest.java | 33 +++++++++ 12 files changed, 253 insertions(+), 90 deletions(-) create mode 100644 sonar-core/src/main/java/org/sonar/core/util/SettingFormatter.java create mode 100644 sonar-core/src/test/java/org/sonar/core/util/SettingFormatterTest.java diff --git a/server/sonar-main/src/main/java/org/sonar/application/config/AppSettingsLoaderImpl.java b/server/sonar-main/src/main/java/org/sonar/application/config/AppSettingsLoaderImpl.java index 5362f8e8818..3ba85f8e606 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/config/AppSettingsLoaderImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/config/AppSettingsLoaderImpl.java @@ -26,30 +26,40 @@ import java.io.InputStreamReader; import java.io.Reader; import java.net.URISyntaxException; import java.util.Arrays; +import java.util.Optional; import java.util.Properties; +import java.util.Set; import java.util.function.Consumer; +import java.util.stream.Collectors; import org.slf4j.LoggerFactory; import org.sonar.core.extension.ServiceLoaderWrapper; import org.sonar.process.ConfigurationUtils; import org.sonar.process.NetworkUtilsImpl; import org.sonar.process.ProcessProperties; import org.sonar.process.Props; +import org.sonar.process.System2; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Optional.ofNullable; +import static org.sonar.core.util.SettingFormatter.fromJavaPropertyToEnvVariable; import static org.sonar.process.ProcessProperties.Property.PATH_HOME; public class AppSettingsLoaderImpl implements AppSettingsLoader { + private final System2 system; private final File homeDir; private final String[] cliArguments; private final Consumer[] consumers; private final ServiceLoaderWrapper serviceLoaderWrapper; - public AppSettingsLoaderImpl(String[] cliArguments, ServiceLoaderWrapper serviceLoaderWrapper) { - this(cliArguments, detectHomeDir(), serviceLoaderWrapper, new FileSystemSettings(), new JdbcSettings(), new ClusterSettings(NetworkUtilsImpl.INSTANCE)); + public AppSettingsLoaderImpl(System2 system, String[] cliArguments, ServiceLoaderWrapper serviceLoaderWrapper) { + this(system, cliArguments, detectHomeDir(), serviceLoaderWrapper, new FileSystemSettings(), new JdbcSettings(), + new ClusterSettings(NetworkUtilsImpl.INSTANCE)); } - AppSettingsLoaderImpl(String[] cliArguments, File homeDir, ServiceLoaderWrapper serviceLoaderWrapper, Consumer... consumers) { + @SafeVarargs + AppSettingsLoaderImpl(System2 system, String[] cliArguments, File homeDir, ServiceLoaderWrapper serviceLoaderWrapper, Consumer... consumers) { + this.system = system; this.cliArguments = cliArguments; this.homeDir = homeDir; this.serviceLoaderWrapper = serviceLoaderWrapper; @@ -63,9 +73,10 @@ public class AppSettingsLoaderImpl implements AppSettingsLoader { @Override public AppSettings load() { Properties p = loadPropertiesFile(homeDir); + fetchSettingsFromEnvironment(system, p); p.putAll(CommandLineParser.parseArguments(cliArguments)); p.setProperty(PATH_HOME.getKey(), homeDir.getAbsolutePath()); - p = ConfigurationUtils.interpolateVariables(p, System.getenv()); + p = ConfigurationUtils.interpolateVariables(p, system.getenv()); // the difference between Properties and Props is that the latter // supports decryption of values, so it must be used when values @@ -77,6 +88,17 @@ public class AppSettingsLoaderImpl implements AppSettingsLoader { return new AppSettingsImpl(props); } + private static void fetchSettingsFromEnvironment(System2 system, Properties properties) { + Set possibleSettings = Arrays.stream(ProcessProperties.Property.values()).map(ProcessProperties.Property::getKey) + .collect(Collectors.toSet()); + possibleSettings.addAll(properties.stringPropertyNames()); + possibleSettings.forEach(key -> { + String environmentVarName = fromJavaPropertyToEnvVariable(key); + Optional envVarValue = ofNullable(system.getenv(environmentVarName)); + envVarValue.ifPresent(value -> properties.put(key, value)); + }); + } + private static File detectHomeDir() { try { File appJar = new File(Class.forName("org.sonar.application.App").getProtectionDomain().getCodeSource().getLocation().toURI()); diff --git a/server/sonar-main/src/test/java/org/sonar/application/config/AppSettingsLoaderImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/config/AppSettingsLoaderImplTest.java index b4c01e36930..988107a6150 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/config/AppSettingsLoaderImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/config/AppSettingsLoaderImplTest.java @@ -19,8 +19,10 @@ */ package org.sonar.application.config; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.io.File; +import java.io.IOException; import org.apache.commons.io.FileUtils; import org.junit.Before; import org.junit.Rule; @@ -28,7 +30,9 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.sonar.core.extension.ServiceLoaderWrapper; +import org.sonar.process.System2; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.data.MapEntry.entry; import static org.mockito.Mockito.mock; @@ -42,6 +46,7 @@ public class AppSettingsLoaderImplTest { public TemporaryFolder temp = new TemporaryFolder(); private ServiceLoaderWrapper serviceLoaderWrapper = mock(ServiceLoaderWrapper.class); + private System2 system = mock(System2.class); @Before public void setup() { @@ -52,20 +57,42 @@ public class AppSettingsLoaderImplTest { public void load_properties_from_file() throws Exception { File homeDir = temp.newFolder(); File propsFile = new File(homeDir, "conf/sonar.properties"); - FileUtils.write(propsFile, "foo=bar"); + FileUtils.write(propsFile, "foo=bar", UTF_8); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[0], homeDir, serviceLoaderWrapper); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(system, new String[0], homeDir, serviceLoaderWrapper); AppSettings settings = underTest.load(); assertThat(settings.getProps().rawProperties()).contains(entry("foo", "bar")); } + @Test + public void load_properties_from_env() throws Exception { + when(system.getenv()).thenReturn(ImmutableMap.of( + "SONAR_DASHED_PROPERTY", "2", + "SONAR_JDBC_URL", "some_jdbc_url", + "SONAR_EMBEDDEDDATABASE_PORT", "8765")); + when(system.getenv("SONAR_DASHED_PROPERTY")).thenReturn("2"); + when(system.getenv("SONAR_JDBC_URL")).thenReturn("some_jdbc_url"); + when(system.getenv("SONAR_EMBEDDEDDATABASE_PORT")).thenReturn("8765"); + File homeDir = temp.newFolder(); + File propsFile = new File(homeDir, "conf/sonar.properties"); + FileUtils.write(propsFile, "sonar.dashed-property=1", UTF_8); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(system, new String[0], homeDir, serviceLoaderWrapper); + + AppSettings settings = underTest.load(); + + assertThat(settings.getProps().rawProperties()).contains( + entry("sonar.dashed-property", "2"), + entry("sonar.jdbc.url", "some_jdbc_url"), + entry("sonar.embeddedDatabase.port", "8765")); + } + @Test public void throws_ISE_if_file_fails_to_be_loaded() throws Exception { File homeDir = temp.newFolder(); File propsFileAsDir = new File(homeDir, "conf/sonar.properties"); FileUtils.forceMkdir(propsFileAsDir); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[0], homeDir, serviceLoaderWrapper); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(system, new String[0], homeDir, serviceLoaderWrapper); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("Cannot open file " + propsFileAsDir.getAbsolutePath()); @@ -77,7 +104,7 @@ public class AppSettingsLoaderImplTest { public void file_is_not_loaded_if_it_does_not_exist() throws Exception { File homeDir = temp.newFolder(); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[0], homeDir, serviceLoaderWrapper); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(system, new String[0], homeDir, serviceLoaderWrapper); AppSettings settings = underTest.load(); // no failure, file is ignored @@ -88,7 +115,7 @@ public class AppSettingsLoaderImplTest { public void command_line_arguments_are_included_to_settings() throws Exception { File homeDir = temp.newFolder(); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[] {"-Dsonar.foo=bar", "-Dhello=world"}, homeDir, serviceLoaderWrapper); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(system, new String[] {"-Dsonar.foo=bar", "-Dhello=world"}, homeDir, serviceLoaderWrapper); AppSettings settings = underTest.load(); assertThat(settings.getProps().rawProperties()) @@ -97,20 +124,47 @@ public class AppSettingsLoaderImplTest { } @Test - public void command_line_arguments_make_precedence_over_properties_files() throws Exception { + public void command_line_arguments_take_precedence_over_properties_files() throws IOException { File homeDir = temp.newFolder(); File propsFile = new File(homeDir, "conf/sonar.properties"); - FileUtils.write(propsFile, "sonar.foo=file"); + FileUtils.write(propsFile, "sonar.foo=file", UTF_8); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[] {"-Dsonar.foo=cli"}, homeDir, serviceLoaderWrapper); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(system, new String[] {"-Dsonar.foo=cli"}, homeDir, serviceLoaderWrapper); AppSettings settings = underTest.load(); assertThat(settings.getProps().rawProperties()).contains(entry("sonar.foo", "cli")); } @Test - public void detectHomeDir_returns_existing_dir() { - assertThat(new AppSettingsLoaderImpl(new String[0], serviceLoaderWrapper).getHomeDir()).exists().isDirectory(); + public void env_vars_take_precedence_over_properties_file() throws Exception { + when(system.getenv()).thenReturn(ImmutableMap.of("SONAR_CUSTOMPROP", "11")); + when(system.getenv("SONAR_CUSTOMPROP")).thenReturn("11"); + File homeDir = temp.newFolder(); + File propsFile = new File(homeDir, "conf/sonar.properties"); + FileUtils.write(propsFile, "sonar.customProp=10", UTF_8); + + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(system, new String[0], homeDir, serviceLoaderWrapper); + AppSettings settings = underTest.load(); + assertThat(settings.getProps().rawProperties()).contains(entry("sonar.customProp", "11")); + } + + @Test + public void command_line_arguments_take_precedence_over_env_vars() throws Exception { + when(system.getenv()).thenReturn(ImmutableMap.of("SONAR_CUSTOMPROP", "11")); + when(system.getenv("SONAR_CUSTOMPROP")).thenReturn("11"); + File homeDir = temp.newFolder(); + File propsFile = new File(homeDir, "conf/sonar.properties"); + FileUtils.write(propsFile, "sonar.customProp=10", UTF_8); + + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(system, new String[] {"-Dsonar.customProp=9"}, homeDir, serviceLoaderWrapper); + AppSettings settings = underTest.load(); + + assertThat(settings.getProps().rawProperties()).contains(entry("sonar.customProp", "9")); + } + + @Test + public void detectHomeDir_returns_existing_dir() { + assertThat(new AppSettingsLoaderImpl(system, new String[0], serviceLoaderWrapper).getHomeDir()).exists().isDirectory(); } } diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java index ae166f57741..12c8d55db7e 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java @@ -66,6 +66,8 @@ public class EsSettingsTest { public ExpectedException expectedException = ExpectedException.none(); private ListAppender listAppender; + private System2 system = mock(System2.class); + @After public void tearDown() { if (listAppender != null) { @@ -77,8 +79,7 @@ public class EsSettingsTest { public void constructor_does_not_logs_warning_if_env_variable_ES_JVM_OPTIONS_is_not_set() { this.listAppender = ListAppender.attachMemoryAppenderToLoggerOf(EsSettings.class); Props props = minimalProps(); - System2 system2 = mock(System2.class); - new EsSettings(props, new EsInstallation(props), system2); + new EsSettings(props, new EsInstallation(props), system); assertThat(listAppender.getLogs()).isEmpty(); } @@ -87,9 +88,8 @@ public class EsSettingsTest { public void constructor_does_not_logs_warning_if_env_variable_ES_JVM_OPTIONS_is_set_and_empty() { this.listAppender = ListAppender.attachMemoryAppenderToLoggerOf(EsSettings.class); Props props = minimalProps(); - System2 system2 = mock(System2.class); - when(system2.getenv("ES_JVM_OPTIONS")).thenReturn(" "); - new EsSettings(props, new EsInstallation(props), system2); + when(system.getenv("ES_JVM_OPTIONS")).thenReturn(" "); + new EsSettings(props, new EsInstallation(props), system); assertThat(listAppender.getLogs()).isEmpty(); } @@ -98,9 +98,8 @@ public class EsSettingsTest { public void constructor_logs_warning_if_env_variable_ES_JVM_OPTIONS_is_set_and_non_empty() { this.listAppender = ListAppender.attachMemoryAppenderToLoggerOf(EsSettings.class); Props props = minimalProps(); - System2 system2 = mock(System2.class); - when(system2.getenv("ES_JVM_OPTIONS")).thenReturn(randomAlphanumeric(2)); - new EsSettings(props, new EsInstallation(props), system2); + when(system.getenv("ES_JVM_OPTIONS")).thenReturn(randomAlphanumeric(2)); + new EsSettings(props, new EsInstallation(props), system); assertThat(listAppender.getLogs()) .extracting(ILoggingEvent::getMessage) @@ -130,7 +129,7 @@ public class EsSettingsTest { props.set(PATH_LOGS.getKey(), temp.newFolder().getAbsolutePath()); props.set(CLUSTER_NAME.getKey(), "sonarqube"); - EsSettings esSettings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); + EsSettings esSettings = new EsSettings(props, new EsInstallation(props), system); Map generated = esSettings.build(); assertThat(generated.get("transport.tcp.port")).isEqualTo("1234"); @@ -169,7 +168,7 @@ public class EsSettingsTest { props.set(Property.CLUSTER_ENABLED.getKey(), "true"); props.set(CLUSTER_NODE_NAME.getKey(), "node-1"); - EsSettings esSettings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); + EsSettings esSettings = new EsSettings(props, new EsInstallation(props), system); Map generated = esSettings.build(); assertThat(generated.get("cluster.name")).isEqualTo("sonarqube-1"); @@ -188,7 +187,7 @@ public class EsSettingsTest { props.set(PATH_DATA.getKey(), temp.newFolder().getAbsolutePath()); props.set(PATH_TEMP.getKey(), temp.newFolder().getAbsolutePath()); props.set(PATH_LOGS.getKey(), temp.newFolder().getAbsolutePath()); - EsSettings esSettings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); + EsSettings esSettings = new EsSettings(props, new EsInstallation(props), system); Map generated = esSettings.build(); assertThat(generated.get("node.name")).startsWith("sonarqube-"); } @@ -205,7 +204,7 @@ public class EsSettingsTest { props.set(PATH_DATA.getKey(), temp.newFolder().getAbsolutePath()); props.set(PATH_TEMP.getKey(), temp.newFolder().getAbsolutePath()); props.set(PATH_LOGS.getKey(), temp.newFolder().getAbsolutePath()); - EsSettings esSettings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); + EsSettings esSettings = new EsSettings(props, new EsInstallation(props), system); Map generated = esSettings.build(); assertThat(generated.get("node.name")).isEqualTo("sonarqube"); } @@ -223,7 +222,7 @@ public class EsSettingsTest { File data = new File(foo, "data"); when(mockedEsInstallation.getDataDirectory()).thenReturn(data); - EsSettings underTest = new EsSettings(minProps(new Random().nextBoolean()), mockedEsInstallation, System2.INSTANCE); + EsSettings underTest = new EsSettings(minProps(new Random().nextBoolean()), mockedEsInstallation, system); Map generated = underTest.build(); assertThat(generated.get("path.data")).isEqualTo(data.getPath()); @@ -235,7 +234,7 @@ public class EsSettingsTest { public void set_discovery_settings_if_cluster_is_enabled() throws Exception { Props props = minProps(CLUSTER_ENABLED); props.set(CLUSTER_SEARCH_HOSTS.getKey(), "1.2.3.4:9000,1.2.3.5:8080"); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("discovery.zen.ping.unicast.hosts")).isEqualTo("1.2.3.4:9000,1.2.3.5:8080"); assertThat(settings.get("discovery.zen.minimum_master_nodes")).isEqualTo("2"); @@ -247,7 +246,7 @@ public class EsSettingsTest { Props props = minProps(CLUSTER_ENABLED); props.set(SEARCH_MINIMUM_MASTER_NODES.getKey(), "ꝱꝲꝳପ"); - EsSettings underTest = new EsSettings(props, new EsInstallation(props), System2.INSTANCE); + EsSettings underTest = new EsSettings(props, new EsInstallation(props), system); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("Value of property sonar.search.minimumMasterNodes is not an integer:"); @@ -258,7 +257,7 @@ public class EsSettingsTest { public void cluster_is_enabled_with_defined_minimum_master_nodes() throws Exception { Props props = minProps(CLUSTER_ENABLED); props.set(SEARCH_MINIMUM_MASTER_NODES.getKey(), "5"); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("discovery.zen.minimum_master_nodes")).isEqualTo("5"); } @@ -267,7 +266,7 @@ public class EsSettingsTest { public void cluster_is_enabled_with_defined_initialTimeout() throws Exception { Props props = minProps(CLUSTER_ENABLED); props.set(SEARCH_INITIAL_STATE_TIMEOUT.getKey(), "10s"); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("discovery.initial_state_timeout")).isEqualTo("10s"); } @@ -276,7 +275,7 @@ public class EsSettingsTest { public void in_standalone_initialTimeout_is_not_overridable() throws Exception { Props props = minProps(CLUSTER_DISABLED); props.set(SEARCH_INITIAL_STATE_TIMEOUT.getKey(), "10s"); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("discovery.initial_state_timeout")).isEqualTo("30s"); } @@ -285,7 +284,7 @@ public class EsSettingsTest { public void in_standalone_minimumMasterNodes_is_not_overridable() throws Exception { Props props = minProps(CLUSTER_DISABLED); props.set(SEARCH_MINIMUM_MASTER_NODES.getKey(), "5"); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("discovery.zen.minimum_master_nodes")).isEqualTo("1"); } @@ -294,7 +293,7 @@ public class EsSettingsTest { public void enable_http_connector() throws Exception { Props props = minProps(CLUSTER_DISABLED); props.set(SEARCH_HTTP_PORT.getKey(), "9010"); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("http.port")).isEqualTo("9010"); assertThat(settings.get("http.host")).isEqualTo("127.0.0.1"); @@ -306,7 +305,7 @@ public class EsSettingsTest { Props props = minProps(CLUSTER_DISABLED); props.set(SEARCH_HTTP_PORT.getKey(), "9010"); props.set(SEARCH_HOST.getKey(), "127.0.0.2"); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("http.port")).isEqualTo("9010"); assertThat(settings.get("http.host")).isEqualTo("127.0.0.2"); @@ -316,7 +315,7 @@ public class EsSettingsTest { @Test public void enable_seccomp_filter_by_default() throws Exception { Props props = minProps(CLUSTER_DISABLED); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("bootstrap.system_call_filter")).isNull(); } @@ -325,7 +324,7 @@ public class EsSettingsTest { public void disable_seccomp_filter_if_configured_in_search_additional_props() throws Exception { Props props = minProps(CLUSTER_DISABLED); props.set("sonar.search.javaAdditionalOpts", "-Xmx1G -Dbootstrap.system_call_filter=false -Dfoo=bar"); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("bootstrap.system_call_filter")).isEqualTo("false"); } @@ -334,7 +333,7 @@ public class EsSettingsTest { public void disable_mmap_if_configured_in_search_additional_props() throws Exception { Props props = minProps(CLUSTER_DISABLED); props.set("sonar.search.javaAdditionalOpts", "-Dnode.store.allow_mmapfs=false"); - Map settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + Map settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("node.store.allow_mmapfs")).isEqualTo("false"); } diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java index fa9a8657aa5..f643c4a744c 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java +++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java @@ -62,6 +62,11 @@ public class ProcessProperties { PATH_TEMP("sonar.path.temp", "temp"), PATH_WEB("sonar.path.web", "web"), + LOG_LEVEL_APP("sonar.log.level.app"), + LOG_LEVEL_WEB("sonar.log.level.web"), + LOG_LEVEL_CE("sonar.log.level.ce"), + LOG_LEVEL_ES("sonar.log.level.es"), + SEARCH_HOST("sonar.search.host", InetAddress.getLoopbackAddress().getHostAddress()), SEARCH_PORT("sonar.search.port", "9001"), SEARCH_HTTP_PORT("sonar.search.httpPort"), diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/setting/ThreadLocalSettings.java b/server/sonar-server-common/src/main/java/org/sonar/server/setting/ThreadLocalSettings.java index 037ccc670a3..f2cf05f92d6 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/setting/ThreadLocalSettings.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/setting/ThreadLocalSettings.java @@ -20,23 +20,26 @@ package org.sonar.server.setting; import com.google.common.annotations.VisibleForTesting; +import java.util.AbstractMap; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.ibatis.exceptions.PersistenceException; import org.sonar.api.CoreProperties; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.config.Encryption; +import org.sonar.api.config.PropertyDefinition; import org.sonar.api.config.PropertyDefinitions; import org.sonar.api.config.Settings; import org.sonar.api.server.ServerSide; -import org.sonar.api.utils.log.Logger; -import org.sonar.api.utils.log.Loggers; +import org.sonar.api.utils.System2; +import org.sonar.core.util.SettingFormatter; -import static java.lang.String.format; import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; @@ -57,27 +60,29 @@ import static java.util.Objects.requireNonNull; @ComputeEngineSide @ServerSide public class ThreadLocalSettings extends Settings { - private static final Logger LOG = Loggers.get(ThreadLocalSettings.class); - - private final Properties overwrittenSystemProps = new Properties(); private final Properties systemProps = new Properties(); + private final Properties corePropsFromEnvVariables = new Properties(); private static final ThreadLocal> CACHE = new ThreadLocal<>(); private Map getPropertyDbFailureCache = Collections.emptyMap(); private Map getPropertiesDbFailureCache = Collections.emptyMap(); private SettingLoader settingLoader; + private System2 system2; - public ThreadLocalSettings(PropertyDefinitions definitions, Properties props) { - this(definitions, props, new NopSettingLoader()); + public ThreadLocalSettings(System2 system2, PropertyDefinitions definitions, Properties props) { + this(system2, definitions, props, new NopSettingLoader()); } @VisibleForTesting - ThreadLocalSettings(PropertyDefinitions definitions, Properties props, SettingLoader settingLoader) { + ThreadLocalSettings(System2 system2, PropertyDefinitions definitions, Properties props, SettingLoader settingLoader) { super(definitions, new Encryption(null)); + this.system2 = system2; this.settingLoader = settingLoader; + + resolveCorePropertiesFromEnvironment(); props.forEach((k, v) -> systemProps.put(k, v == null ? null : v.toString().trim())); // TODO something wrong about lifecycle here. It could be improved - getEncryption().setPathToSecretKey(props.getProperty(CoreProperties.ENCRYPTION_SECRET_KEY_PATH)); + getEncryption().setPathToSecretKey(get(CoreProperties.ENCRYPTION_SECRET_KEY_PATH).orElse(null)); } @VisibleForTesting @@ -92,17 +97,17 @@ public class ThreadLocalSettings extends Settings { @Override protected Optional get(String key) { // search for the first value available in - // 1. overwritten system properties - // 2. system properties + // 1. system properties + // 2. core property from environment variable // 3. thread local cache (if enabled) // 4. db - String value = overwrittenSystemProps.getProperty(key); + String value = systemProps.getProperty(key); if (value != null) { return Optional.of(value); } - value = systemProps.getProperty(key); + value = corePropsFromEnvVariables.getProperty(key); if (value != null) { return Optional.of(value); } @@ -135,15 +140,6 @@ public class ThreadLocalSettings extends Settings { } } - public void setSystemProperty(String key, String value) { - checkKeyAndValue(key, value); - String systemValue = systemProps.getProperty(key); - if (LOG.isDebugEnabled() && systemValue != null && !value.equals(systemValue)) { - LOG.debug(format("System property '%s' with value '%s' overwritten with value '%s'", key, systemValue, value)); - } - overwrittenSystemProps.put(key, value.trim()); - } - @Override protected void set(String key, String value) { checkKeyAndValue(key, value); @@ -187,6 +183,7 @@ public class ThreadLocalSettings extends Settings { public Map getProperties() { Map result = new HashMap<>(); loadAll(result); + corePropsFromEnvVariables.forEach((k, v) -> result.put((String) k, (String) v)); systemProps.forEach((key, value) -> result.put((String) key, (String) value)); return unmodifiableMap(result); } @@ -200,4 +197,20 @@ public class ThreadLocalSettings extends Settings { appendTo.putAll(getPropertiesDbFailureCache); } } + + private void resolveCorePropertiesFromEnvironment() { + corePropsFromEnvVariables.putAll(this.getDefinitions().getAll() + .stream() + .map(PropertyDefinition::key) + .flatMap(p -> { + String envVar = SettingFormatter.fromJavaPropertyToEnvVariable(p); + String envVarValue = system2.envVariable(envVar); + if (envVarValue != null) { + return Stream.of(new AbstractMap.SimpleEntry<>(p, envVarValue)); + } else { + return Stream.empty(); + } + }) + .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue))); + } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/setting/ThreadLocalSettingsTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/setting/ThreadLocalSettingsTest.java index ddcfef16827..90624314c4a 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/setting/ThreadLocalSettingsTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/setting/ThreadLocalSettingsTest.java @@ -34,6 +34,8 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.sonar.api.config.PropertyDefinitions; +import org.sonar.api.utils.System2; +import org.sonar.core.config.CorePropertyDefinitions; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -44,6 +46,7 @@ import static org.assertj.core.data.MapEntry.entry; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ThreadLocalSettingsTest { @@ -56,6 +59,7 @@ public class ThreadLocalSettingsTest { private MapSettingLoader dbSettingLoader = new MapSettingLoader(); private ThreadLocalSettings underTest = null; + private System2 system = mock(System2.class); @After public void tearDown() { @@ -141,7 +145,7 @@ public class ThreadLocalSettingsTest { private ThreadLocalSettings create(Map systemProps) { Properties p = new Properties(); p.putAll(systemProps); - return new ThreadLocalSettings(new PropertyDefinitions(), p, dbSettingLoader); + return new ThreadLocalSettings(system, new PropertyDefinitions(CorePropertyDefinitions.all()), p, dbSettingLoader); } @Test @@ -153,6 +157,16 @@ public class ThreadLocalSettingsTest { assertThat(underTest.getProperties()).containsOnly(entry("foo", "1"), entry("bar", "2")); } + @Test + public void load_core_properties_from_environment() { + when(system.envVariable("SONAR_FORCEAUTHENTICATION")).thenReturn("true"); + underTest = create(ImmutableMap.of()); + + assertThat(underTest.get("sonar.forceAuthentication")).hasValue("true"); + assertThat(underTest.get("missing")).isNotPresent(); + assertThat(underTest.getProperties()).containsOnly(entry("sonar.forceAuthentication", "true")); + } + @Test public void database_properties_are_not_cached_by_default() { insertPropertyIntoDb("foo", "from db"); @@ -165,25 +179,6 @@ public class ThreadLocalSettingsTest { assertThat(underTest.get("foo")).isNotPresent(); } - @Test - public void overwritten_system_settings_have_precedence_over_system_and_databse() { - underTest = create(ImmutableMap.of("foo", "from system")); - - underTest.setSystemProperty("foo", "donut"); - - assertThat(underTest.get("foo")).hasValue("donut"); - } - - @Test - public void overwritten_system_settings_have_precedence_over_databse() { - insertPropertyIntoDb("foo", "from db"); - underTest = create(Collections.emptyMap()); - - underTest.setSystemProperty("foo", "donut"); - - assertThat(underTest.get("foo")).hasValue("donut"); - } - @Test public void system_settings_have_precedence_over_database() { insertPropertyIntoDb("foo", "from db"); @@ -268,7 +263,7 @@ public class ThreadLocalSettingsTest { @Test public void change_setting_loader() { - underTest = new ThreadLocalSettings(new PropertyDefinitions(), new Properties()); + underTest = new ThreadLocalSettings(system, new PropertyDefinitions(), new Properties()); assertThat(underTest.getSettingLoader()).isNotNull(); @@ -291,7 +286,7 @@ public class ThreadLocalSettingsTest { SettingLoader settingLoaderMock = mock(SettingLoader.class); PersistenceException toBeThrown = new PersistenceException("Faking an error connecting to DB"); doThrow(toBeThrown).when(settingLoaderMock).loadAll(); - underTest = new ThreadLocalSettings(new PropertyDefinitions(), new Properties(), settingLoaderMock); + underTest = new ThreadLocalSettings(system, new PropertyDefinitions(), new Properties(), settingLoaderMock); assertThat(underTest.getProperties()) .isEmpty(); @@ -302,7 +297,7 @@ public class ThreadLocalSettingsTest { SettingLoader settingLoaderMock = mock(SettingLoader.class); PersistenceException toBeThrown = new PersistenceException("Faking an error connecting to DB"); doThrow(toBeThrown).when(settingLoaderMock).loadAll(); - underTest = new ThreadLocalSettings(new PropertyDefinitions(), new Properties(), settingLoaderMock); + underTest = new ThreadLocalSettings(system, new PropertyDefinitions(), new Properties(), settingLoaderMock); underTest.load(); assertThat(underTest.getProperties()) @@ -321,7 +316,7 @@ public class ThreadLocalSettingsTest { .doAnswer(invocationOnMock -> ImmutableMap.of(key, value2)) .when(settingLoaderMock) .loadAll(); - underTest = new ThreadLocalSettings(new PropertyDefinitions(), new Properties(), settingLoaderMock); + underTest = new ThreadLocalSettings(system, new PropertyDefinitions(), new Properties(), settingLoaderMock); underTest.load(); assertThat(underTest.getProperties()) @@ -345,7 +340,7 @@ public class ThreadLocalSettingsTest { PersistenceException toBeThrown = new PersistenceException("Faking an error connecting to DB"); String key = randomAlphanumeric(3); doThrow(toBeThrown).when(settingLoaderMock).load(key); - underTest = new ThreadLocalSettings(new PropertyDefinitions(), new Properties(), settingLoaderMock); + underTest = new ThreadLocalSettings(system, new PropertyDefinitions(), new Properties(), settingLoaderMock); assertThat(underTest.get(key)).isEmpty(); } @@ -356,7 +351,7 @@ public class ThreadLocalSettingsTest { PersistenceException toBeThrown = new PersistenceException("Faking an error connecting to DB"); String key = randomAlphanumeric(3); doThrow(toBeThrown).when(settingLoaderMock).load(key); - underTest = new ThreadLocalSettings(new PropertyDefinitions(), new Properties(), settingLoaderMock); + underTest = new ThreadLocalSettings(system, new PropertyDefinitions(), new Properties(), settingLoaderMock); underTest.load(); assertThat(underTest.get(key)).isEmpty(); diff --git a/sonar-application/src/main/assembly/conf/sonar.properties b/sonar-application/src/main/assembly/conf/sonar.properties index b7067153c6a..f26ac701236 100644 --- a/sonar-application/src/main/assembly/conf/sonar.properties +++ b/sonar-application/src/main/assembly/conf/sonar.properties @@ -1,5 +1,7 @@ # Property values can: -# - reference an environment variable, for example sonar.jdbc.url= ${env:SONAR_JDBC_URL} +# - be overridden by environment variables. The name of the corresponding environment variable is the +# upper-cased name of the property where all the dot ('.') and dash ('-') characters are replaced by +# underscores ('_'). For example, to override 'sonar.web.systemPasscode' use 'SONAR_WEB_SYSTEMPASSCODE'. # - be encrypted. See https://redirect.sonarsource.com/doc/settings-encryption.html #-------------------------------------------------------------------------------------------------- diff --git a/sonar-application/src/main/java/org/sonar/application/App.java b/sonar-application/src/main/java/org/sonar/application/App.java index 0260d25ca5b..5ab3ae12142 100644 --- a/sonar-application/src/main/java/org/sonar/application/App.java +++ b/sonar-application/src/main/java/org/sonar/application/App.java @@ -42,12 +42,13 @@ public class App { private final JavaVersion javaVersion; private StopRequestWatcher stopRequestWatcher = null; private StopRequestWatcher hardStopRequestWatcher = null; + public App(JavaVersion javaVersion) { this.javaVersion = javaVersion; } public void start(String[] cliArguments) { - AppSettingsLoader settingsLoader = new AppSettingsLoaderImpl(cliArguments, new ServiceLoaderWrapper()); + AppSettingsLoader settingsLoader = new AppSettingsLoaderImpl(System2.INSTANCE, cliArguments, new ServiceLoaderWrapper()); AppSettings settings = settingsLoader.load(); // order is important - logging must be configured before any other components (AppFileSystem, ...) AppLogging logging = new AppLogging(settings); diff --git a/sonar-core/src/main/java/org/sonar/core/config/CorePropertyDefinitions.java b/sonar-core/src/main/java/org/sonar/core/config/CorePropertyDefinitions.java index 8eccefd97b1..295cc2fdaf7 100644 --- a/sonar-core/src/main/java/org/sonar/core/config/CorePropertyDefinitions.java +++ b/sonar-core/src/main/java/org/sonar/core/config/CorePropertyDefinitions.java @@ -29,6 +29,7 @@ import org.sonar.api.resources.Qualifiers; import static java.util.Arrays.asList; import static org.sonar.api.PropertyType.BOOLEAN; +import static org.sonar.api.PropertyType.STRING; public class CorePropertyDefinitions { @@ -70,6 +71,12 @@ public class CorePropertyDefinitions { .category(CoreProperties.CATEGORY_GENERAL) .build(), + PropertyDefinition.builder(CoreProperties.ENCRYPTION_SECRET_KEY_PATH) + .name("Encryption secret key path") + .description("Path to a file that contains encryption secret key that is used to encrypting other settings.") + .type(STRING) + .hidden() + .build(), PropertyDefinition.builder("sonar.authenticator.downcase") .name("Downcase login") .description("Downcase login during user authentication, typically for Active Directory") diff --git a/sonar-core/src/main/java/org/sonar/core/util/SettingFormatter.java b/sonar-core/src/main/java/org/sonar/core/util/SettingFormatter.java new file mode 100644 index 00000000000..7233a79cbbb --- /dev/null +++ b/sonar-core/src/main/java/org/sonar/core/util/SettingFormatter.java @@ -0,0 +1,32 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.core.util; + +import java.util.Locale; + +public final class SettingFormatter { + private SettingFormatter() { + // util class + } + + public static String fromJavaPropertyToEnvVariable(String property) { + return property.toUpperCase(Locale.ENGLISH).replace('.', '_').replace('-', '_'); + } +} diff --git a/sonar-core/src/test/java/org/sonar/core/config/CorePropertyDefinitionsTest.java b/sonar-core/src/test/java/org/sonar/core/config/CorePropertyDefinitionsTest.java index 50d281dc494..e03f5d7868d 100644 --- a/sonar-core/src/test/java/org/sonar/core/config/CorePropertyDefinitionsTest.java +++ b/sonar-core/src/test/java/org/sonar/core/config/CorePropertyDefinitionsTest.java @@ -30,7 +30,7 @@ public class CorePropertyDefinitionsTest { @Test public void all() { List defs = CorePropertyDefinitions.all(); - assertThat(defs).hasSize(50); + assertThat(defs).hasSize(51); } @Test diff --git a/sonar-core/src/test/java/org/sonar/core/util/SettingFormatterTest.java b/sonar-core/src/test/java/org/sonar/core/util/SettingFormatterTest.java new file mode 100644 index 00000000000..d9aed76edf9 --- /dev/null +++ b/sonar-core/src/test/java/org/sonar/core/util/SettingFormatterTest.java @@ -0,0 +1,33 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.core.util; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SettingFormatterTest { + + @Test + public void fromJavaPropertyToEnvVariable() { + String output = SettingFormatter.fromJavaPropertyToEnvVariable("some.randomProperty-123.test"); + assertThat(output).isEqualTo("SOME_RANDOMPROPERTY_123_TEST"); + } +} -- 2.39.5