diff options
author | Michal Duda <michal.duda@sonarsource.com> | 2019-05-16 12:36:17 +0200 |
---|---|---|
committer | SonarTech <sonartech@sonarsource.com> | 2019-05-21 20:21:07 +0200 |
commit | dfcb1c01a430cf3317d12ff4448fc700b9979d4f (patch) | |
tree | 07a32c098cc59661abf85ff15bdff909e2cfd4b0 /server | |
parent | 42aa152b7b174ce55796b8b00758d1fe63643cb7 (diff) | |
download | sonarqube-dfcb1c01a430cf3317d12ff4448fc700b9979d4f.tar.gz sonarqube-dfcb1c01a430cf3317d12ff4448fc700b9979d4f.zip |
SONAR-11720 Set different memory defaults for EE+
Diffstat (limited to 'server')
9 files changed, 191 insertions, 30 deletions
diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java index 3494bc34fc6..62ea3f86c44 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java @@ -19,6 +19,7 @@ */ package org.sonar.ce.container; +import com.google.common.collect.ImmutableSet; import java.io.File; import java.io.IOException; import java.util.Date; @@ -38,6 +39,7 @@ import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; import org.sonar.ce.CeDistributedInformationImpl; import org.sonar.ce.StandaloneCeDistributedInformation; +import org.sonar.core.extension.ServiceLoaderWrapper; import org.sonar.db.DbTester; import org.sonar.db.property.PropertyDto; import org.sonar.process.ProcessId; @@ -48,6 +50,7 @@ import org.sonar.server.property.InternalProperties; import static java.lang.String.valueOf; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.sonar.process.ProcessEntryPoint.PROPERTY_PROCESS_INDEX; import static org.sonar.process.ProcessEntryPoint.PROPERTY_SHARED_PATH; import static org.sonar.process.ProcessProperties.Property.JDBC_PASSWORD; @@ -68,9 +71,12 @@ public class ComputeEngineContainerImplTest { public DbTester db = DbTester.create(System2.INSTANCE); private ComputeEngineContainerImpl underTest; + private ServiceLoaderWrapper serviceLoaderWrapper = mock(ServiceLoaderWrapper.class); + private ProcessProperties processProperties = new ProcessProperties(serviceLoaderWrapper); @Before public void setUp() { + when(serviceLoaderWrapper.load()).thenReturn(ImmutableSet.of()); underTest = new ComputeEngineContainerImpl(); underTest.setComputeEngineStatus(mock(ComputeEngineStatus.class)); } @@ -148,7 +154,10 @@ public class ComputeEngineContainerImplTest { } private Properties getProperties() throws IOException { - Properties properties = ProcessProperties.defaults(); + Properties properties = new Properties(); + Props props = new Props(properties); + processProperties.completeDefaults(props); + properties = props.rawProperties(); File homeDir = tempFolder.newFolder(); File dataDir = new File(homeDir, "data"); dataDir.mkdirs(); diff --git a/server/sonar-main/src/main/java/org/sonar/application/config/AppSettingsLoaderImpl.java b/server/sonar-main/src/main/java/org/sonar/application/config/AppSettingsLoaderImpl.java index 92811eb78da..64f5a32c380 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/config/AppSettingsLoaderImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/config/AppSettingsLoaderImpl.java @@ -29,12 +29,13 @@ import java.util.Arrays; import java.util.Properties; import java.util.function.Consumer; import org.slf4j.LoggerFactory; +import org.sonar.core.extension.ServiceLoaderWrapper; import org.sonar.process.ConfigurationUtils; import org.sonar.process.NetworkUtilsImpl; +import org.sonar.process.ProcessProperties; import org.sonar.process.Props; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.sonar.process.ProcessProperties.completeDefaults; import static org.sonar.process.ProcessProperties.Property.PATH_HOME; public class AppSettingsLoaderImpl implements AppSettingsLoader { @@ -42,15 +43,16 @@ public class AppSettingsLoaderImpl implements AppSettingsLoader { private final File homeDir; private final String[] cliArguments; private final Consumer<Props>[] consumers; + private final ServiceLoaderWrapper serviceLoaderWrapper; - public AppSettingsLoaderImpl(String[] cliArguments) { - this(cliArguments, detectHomeDir(), - new FileSystemSettings(), new JdbcSettings(), new ClusterSettings(NetworkUtilsImpl.INSTANCE)); + public AppSettingsLoaderImpl(String[] cliArguments, ServiceLoaderWrapper serviceLoaderWrapper) { + this(cliArguments, detectHomeDir(), serviceLoaderWrapper, new FileSystemSettings(), new JdbcSettings(), new ClusterSettings(NetworkUtilsImpl.INSTANCE)); } - AppSettingsLoaderImpl(String[] cliArguments, File homeDir, Consumer<Props>... consumers) { + AppSettingsLoaderImpl(String[] cliArguments, File homeDir, ServiceLoaderWrapper serviceLoaderWrapper, Consumer<Props>... consumers) { this.cliArguments = cliArguments; this.homeDir = homeDir; + this.serviceLoaderWrapper = serviceLoaderWrapper; this.consumers = consumers; } @@ -69,7 +71,7 @@ public class AppSettingsLoaderImpl implements AppSettingsLoader { // supports decryption of values, so it must be used when values // are accessed Props props = new Props(p); - completeDefaults(props); + new ProcessProperties(serviceLoaderWrapper).completeDefaults(props); Arrays.stream(consumers).forEach(c -> c.accept(props)); return new AppSettingsImpl(props); diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java index 1f3a059a34c..babc1e2204b 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java @@ -20,6 +20,7 @@ package org.sonar.application.command; import ch.qos.logback.classic.spi.ILoggingEvent; +import com.google.common.collect.ImmutableSet; import java.io.File; import java.io.IOException; import java.util.Properties; @@ -33,6 +34,7 @@ import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; import org.sonar.application.es.EsInstallation; import org.sonar.application.logging.ListAppender; +import org.sonar.core.extension.ServiceLoaderWrapper; import org.sonar.process.ProcessId; import org.sonar.process.ProcessProperties; import org.sonar.process.Props; @@ -40,6 +42,7 @@ import org.sonar.process.System2; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class CommandFactoryImplTest { @@ -320,7 +323,9 @@ public class CommandFactoryImplTest { p.putAll(userProps); Props props = new Props(p); - ProcessProperties.completeDefaults(props); + ServiceLoaderWrapper serviceLoaderWrapper = mock(ServiceLoaderWrapper.class); + when(serviceLoaderWrapper.load()).thenReturn(ImmutableSet.of()); + new ProcessProperties(serviceLoaderWrapper).completeDefaults(props); return new CommandFactoryImpl(props, tempDir, system2, javaVersion); } diff --git a/server/sonar-main/src/test/java/org/sonar/application/config/AppSettingsLoaderImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/config/AppSettingsLoaderImplTest.java index 482040aef5d..2aaa3eb458b 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/config/AppSettingsLoaderImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/config/AppSettingsLoaderImplTest.java @@ -19,15 +19,20 @@ */ package org.sonar.application.config; +import com.google.common.collect.ImmutableSet; import java.io.File; import org.apache.commons.io.FileUtils; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.sonar.core.extension.ServiceLoaderWrapper; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.data.MapEntry.entry; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class AppSettingsLoaderImplTest { @@ -36,13 +41,20 @@ public class AppSettingsLoaderImplTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); + private ServiceLoaderWrapper serviceLoaderWrapper = mock(ServiceLoaderWrapper.class); + + @Before + public void setup() { + when(serviceLoaderWrapper.load()).thenReturn(ImmutableSet.of()); + } + @Test public void load_properties_from_file() throws Exception { File homeDir = temp.newFolder(); File propsFile = new File(homeDir, "conf/sonar.properties"); FileUtils.write(propsFile, "foo=bar"); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[0], homeDir); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[0], homeDir, serviceLoaderWrapper); AppSettings settings = underTest.load(); assertThat(settings.getProps().rawProperties()).contains(entry("foo", "bar")); @@ -53,7 +65,7 @@ public class AppSettingsLoaderImplTest { File homeDir = temp.newFolder(); File propsFileAsDir = new File(homeDir, "conf/sonar.properties"); FileUtils.forceMkdir(propsFileAsDir); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[0], homeDir); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[0], homeDir, serviceLoaderWrapper); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("Cannot open file " + propsFileAsDir.getAbsolutePath()); @@ -65,7 +77,7 @@ public class AppSettingsLoaderImplTest { public void file_is_not_loaded_if_it_does_not_exist() throws Exception { File homeDir = temp.newFolder(); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[0], homeDir); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[0], homeDir, serviceLoaderWrapper); AppSettings settings = underTest.load(); // no failure, file is ignored @@ -76,7 +88,7 @@ public class AppSettingsLoaderImplTest { public void command_line_arguments_are_included_to_settings() throws Exception { File homeDir = temp.newFolder(); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[] {"-Dsonar.foo=bar", "-Dhello=world"}, homeDir); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[] {"-Dsonar.foo=bar", "-Dhello=world"}, homeDir, serviceLoaderWrapper); AppSettings settings = underTest.load(); assertThat(settings.getProps().rawProperties()) @@ -90,7 +102,7 @@ public class AppSettingsLoaderImplTest { File propsFile = new File(homeDir, "conf/sonar.properties"); FileUtils.write(propsFile, "sonar.foo=file"); - AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[]{"-Dsonar.foo=cli"}, homeDir); + AppSettingsLoaderImpl underTest = new AppSettingsLoaderImpl(new String[] {"-Dsonar.foo=cli"}, homeDir, serviceLoaderWrapper); AppSettings settings = underTest.load(); assertThat(settings.getProps().rawProperties()).contains(entry("sonar.foo", "cli")); @@ -98,7 +110,7 @@ public class AppSettingsLoaderImplTest { @Test public void detectHomeDir_returns_existing_dir() { - assertThat(new AppSettingsLoaderImpl(new String[0]).getHomeDir()).exists().isDirectory(); + assertThat(new AppSettingsLoaderImpl(new String[0], serviceLoaderWrapper).getHomeDir()).exists().isDirectory(); } } diff --git a/server/sonar-main/src/test/java/org/sonar/application/config/TestAppSettings.java b/server/sonar-main/src/test/java/org/sonar/application/config/TestAppSettings.java index 0f319b2371a..ad3fcf195df 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/config/TestAppSettings.java +++ b/server/sonar-main/src/test/java/org/sonar/application/config/TestAppSettings.java @@ -19,11 +19,16 @@ */ package org.sonar.application.config; +import com.google.common.collect.ImmutableSet; import java.util.Optional; import java.util.Properties; +import org.sonar.core.extension.ServiceLoaderWrapper; import org.sonar.process.ProcessProperties; import org.sonar.process.Props; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + /** * Simple implementation of {@link AppSettings} that loads * the default values defined by {@link ProcessProperties}. @@ -34,7 +39,9 @@ public class TestAppSettings implements AppSettings { public TestAppSettings() { this.props = new Props(new Properties()); - ProcessProperties.completeDefaults(this.props); + ServiceLoaderWrapper serviceLoaderWrapper = mock(ServiceLoaderWrapper.class); + when(serviceLoaderWrapper.load()).thenReturn(ImmutableSet.of()); + new ProcessProperties(serviceLoaderWrapper).completeDefaults(this.props); } public TestAppSettings set(String key, String value) { diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java index 8d4a0ab1afc..b859dfd9881 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java @@ -20,6 +20,7 @@ package org.sonar.application.es; import ch.qos.logback.classic.spi.ILoggingEvent; +import com.google.common.collect.ImmutableSet; import java.io.File; import java.io.IOException; import java.util.Map; @@ -31,6 +32,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.sonar.application.logging.ListAppender; +import org.sonar.core.extension.ServiceLoaderWrapper; import org.sonar.process.ProcessProperties; import org.sonar.process.ProcessProperties.Property; import org.sonar.process.Props; @@ -331,7 +333,9 @@ public class EsSettingsTest { private Props minProps(boolean cluster) throws IOException { File homeDir = temp.newFolder(); Props props = new Props(new Properties()); - ProcessProperties.completeDefaults(props); + ServiceLoaderWrapper serviceLoaderWrapper = mock(ServiceLoaderWrapper.class); + when(serviceLoaderWrapper.load()).thenReturn(ImmutableSet.of()); + new ProcessProperties(serviceLoaderWrapper).completeDefaults(props); props.set(PATH_HOME.getKey(), homeDir.getAbsolutePath()); props.set(Property.CLUSTER_ENABLED.getKey(), Boolean.toString(cluster)); return props; diff --git a/server/sonar-process/build.gradle b/server/sonar-process/build.gradle index 4f6c00cbf54..d458ec793f0 100644 --- a/server/sonar-process/build.gradle +++ b/server/sonar-process/build.gradle @@ -15,7 +15,8 @@ dependencies { compile 'com.hazelcast:hazelcast' compile 'org.slf4j:jul-to-slf4j' compile 'org.slf4j:slf4j-api' - + compile project(':sonar-core') + compileOnly 'com.google.code.findbugs:jsr305' compileOnly 'com.google.protobuf:protobuf-java' compileOnly 'org.nanohttpd:nanohttpd' diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java index cf9dce23bd5..fd0a8b7dad2 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java +++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java @@ -22,12 +22,18 @@ package org.sonar.process; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Properties; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.sonar.core.extension.CoreExtension; +import org.sonar.core.extension.ServiceLoaderWrapper; + +import static java.lang.String.format; /** * Constants shared by search, web server and app processes. @@ -35,6 +41,8 @@ import javax.annotation.Nullable; */ public class ProcessProperties { + private final ServiceLoaderWrapper serviceLoaderWrapper; + public enum Property { JDBC_URL("sonar.jdbc.url"), JDBC_USERNAME("sonar.jdbc.username", ""), @@ -151,11 +159,11 @@ public class ProcessProperties { } } - private ProcessProperties() { - // only static stuff + public ProcessProperties(ServiceLoaderWrapper serviceLoaderWrapper) { + this.serviceLoaderWrapper = serviceLoaderWrapper; } - public static void completeDefaults(Props props) { + public void completeDefaults(Props props) { // init string properties for (Map.Entry<Object, Object> entry : defaults().entrySet()) { props.setDefault(entry.getKey().toString(), entry.getValue().toString()); @@ -164,14 +172,30 @@ public class ProcessProperties { fixPortIfZero(props, Property.SEARCH_HOST.getKey(), Property.SEARCH_PORT.getKey()); } - public static Properties defaults() { + private Properties defaults() { Properties defaults = new Properties(); defaults.putAll(Arrays.stream(Property.values()) .filter(Property::hasDefaultValue) .collect(Collectors.toMap(Property::getKey, Property::getDefaultValue))); + defaults.putAll(loadDefaultsFromExtensions()); return defaults; } + private Map<String, String> loadDefaultsFromExtensions() { + Map<String, String> propertyDefaults = new HashMap<>(); + Set<CoreExtension> extensions = serviceLoaderWrapper.load(); + for (CoreExtension ext : extensions) { + for (Map.Entry<String, String> property : ext.getExtensionProperties().entrySet()) { + if (propertyDefaults.put(property.getKey(), property.getValue()) != null) { + throw new IllegalStateException(format("Configuration error: property definition named '%s' found in multiple extensions.", + property.getKey())); + } + } + } + + return propertyDefaults; + } + private static void fixPortIfZero(Props props, String addressPropertyKey, String portPropertyKey) { String port = props.value(portPropertyKey); if ("0".equals(port)) { diff --git a/server/sonar-process/src/test/java/org/sonar/process/ProcessPropertiesTest.java b/server/sonar-process/src/test/java/org/sonar/process/ProcessPropertiesTest.java index 24b0f49c810..7a5d004249c 100644 --- a/server/sonar-process/src/test/java/org/sonar/process/ProcessPropertiesTest.java +++ b/server/sonar-process/src/test/java/org/sonar/process/ProcessPropertiesTest.java @@ -19,19 +19,34 @@ */ package org.sonar.process; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import java.net.InetAddress; +import java.util.Map; import java.util.Properties; +import org.junit.Rule; import org.junit.Test; -import org.sonar.test.TestUtils; +import org.junit.rules.ExpectedException; +import org.sonar.core.extension.CoreExtension; +import org.sonar.core.extension.ServiceLoaderWrapper; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ProcessPropertiesTest { + private ServiceLoaderWrapper serviceLoaderWrapper = mock(ServiceLoaderWrapper.class); + private ProcessProperties processProperties = new ProcessProperties(serviceLoaderWrapper); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Test public void completeDefaults_adds_default_values() { Props props = new Props(new Properties()); - ProcessProperties.completeDefaults(props); + + processProperties.completeDefaults(props); assertThat(props.value("sonar.search.javaOpts")).contains("-Xmx"); assertThat(props.valueAsInt("sonar.jdbc.maxActive")).isEqualTo(60); @@ -44,21 +59,22 @@ public class ProcessPropertiesTest { Properties p = new Properties(); p.setProperty("sonar.jdbc.username", "angela"); Props props = new Props(p); - ProcessProperties.completeDefaults(props); + + processProperties.completeDefaults(props); assertThat(props.value("sonar.jdbc.username")).isEqualTo("angela"); } @Test - public void completeDefaults_set_default_elasticsearch_port_and_bind_address() throws Exception{ + public void completeDefaults_set_default_elasticsearch_port_and_bind_address() throws Exception { Properties p = new Properties(); Props props = new Props(p); - ProcessProperties.completeDefaults(props); + + processProperties.completeDefaults(props); String address = props.value("sonar.search.host"); assertThat(address).isNotEmpty(); assertThat(InetAddress.getByName(address).isLoopbackAddress()).isTrue(); - assertThat(props.valueAsInt("sonar.search.port")).isEqualTo(9001); } @@ -68,12 +84,93 @@ public class ProcessPropertiesTest { p.setProperty("sonar.search.port", "0"); Props props = new Props(p); - ProcessProperties.completeDefaults(props); + processProperties.completeDefaults(props); + assertThat(props.valueAsInt("sonar.search.port")).isGreaterThan(0); } @Test - public void private_constructor() { - assertThat(TestUtils.hasOnlyPrivateConstructors(ProcessProperties.class)).isTrue(); + public void defaults_loads_properties_defaults_from_base_and_extensions() { + Props p = new Props(new Properties()); + when(serviceLoaderWrapper.load()).thenReturn(ImmutableSet.of(new FakeExtension1(), new FakeExtension3())); + + processProperties.completeDefaults(p); + + assertThat(p.value("sonar.some.property")).isEqualTo("1"); + assertThat(p.value("sonar.some.property2")).isEqualTo("455"); + assertThat(p.value("sonar.some.property4")).isEqualTo("abc"); + assertThat(p.value("sonar.some.property5")).isEqualTo("def"); + assertThat(p.value("sonar.some.property5")).isEqualTo("def"); + assertThat(p.value("sonar.search.port")).isEqualTo("9001"); + } + + @Test + public void defaults_throws_exception_on_same_property_defined_more_than_once_in_extensions() { + Props p = new Props(new Properties()); + when(serviceLoaderWrapper.load()).thenReturn(ImmutableSet.of(new FakeExtension1(), new FakeExtension2())); + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Configuration error: property definition named 'sonar.some.property2' found in multiple extensions."); + + processProperties.completeDefaults(p); + } + + private class FakeExtension1 implements CoreExtension { + + @Override + public String getName() { + return "fakeExt1"; + } + + @Override + public void load(Context context) { + // do nothing + } + + @Override + public Map<String, String> getExtensionProperties() { + return ImmutableMap.of( + "sonar.some.property", "1", + "sonar.some.property2", "455"); + } + } + + private class FakeExtension2 implements CoreExtension { + + @Override + public String getName() { + return "fakeExt2"; + } + + @Override + public void load(Context context) { + // do nothing + } + + @Override + public Map<String, String> getExtensionProperties() { + return ImmutableMap.of( + "sonar.some.property2", "5435", + "sonar.some.property3", "32131"); + } + } + + private class FakeExtension3 implements CoreExtension { + + @Override + public String getName() { + return "fakeExt3"; + } + + @Override + public void load(Context context) { + // do nothing + } + + @Override + public Map<String, String> getExtensionProperties() { + return ImmutableMap.of( + "sonar.some.property4", "abc", + "sonar.some.property5", "def"); + } } } |