From d95bc1cd4454a7bd1e16ffc15fba14546c698909 Mon Sep 17 00:00:00 2001 From: Eric Hartmann Date: Mon, 14 Aug 2017 16:44:35 +0200 Subject: [PATCH] SONAR-9712 reduce types of cluster nodes to only: application OR search --- .../ComputeEngineContainerImplTest.java | 1 + .../application/config/ClusterSettings.java | 55 ++++++++----- .../sonar/application/SchedulerImplTest.java | 74 +++++++++++------ .../config/ClusterSettingsLoopbackTest.java | 2 + .../config/ClusterSettingsTest.java | 53 ++++++++++--- .../main/java/org/sonar/process/NodeType.java | 53 +++++++++++++ .../org/sonar/process/ProcessProperties.java | 7 +- .../org/sonar/server/es/EsClientProvider.java | 10 ++- .../sonar/server/es/EsClientProviderTest.java | 7 +- .../org/sonarqube/tests/cluster/Cluster.java | 79 ++++++++++--------- .../tests/cluster/DataCenterEdition.java | 20 ++--- .../tests/cluster/DataCenterEditionTest.java | 32 ++++++++ 12 files changed, 278 insertions(+), 115 deletions(-) create mode 100644 server/sonar-process/src/main/java/org/sonar/process/NodeType.java diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java index 6d4f3e9514a..89f8f852c59 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java @@ -78,6 +78,7 @@ public class ComputeEngineContainerImplTest { HazelcastInstance hzInstance = HazelcastTestHelper.createHazelcastCluster(CLUSTER_NAME, port); Properties properties = getProperties(); + properties.setProperty(ProcessProperties.CLUSTER_NODE_TYPE, "application"); properties.setProperty(ProcessProperties.CLUSTER_ENABLED, "true"); properties.setProperty(ProcessProperties.CLUSTER_LOCALENDPOINT, String.format("%s:%d", hzInstance.getCluster().getLocalMember().getAddress().getHost(), port)); properties.setProperty(ProcessProperties.CLUSTER_NAME, CLUSTER_NAME); diff --git a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java index b6b8ce39128..3861936d858 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java +++ b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java @@ -26,23 +26,27 @@ import java.net.InetAddress; import java.net.NetworkInterface; import java.net.SocketException; import java.net.UnknownHostException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Consumer; +import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; import org.sonar.process.MessageException; +import org.sonar.process.NodeType; import org.sonar.process.ProcessId; import org.sonar.process.ProcessProperties; import org.sonar.process.Props; import static com.google.common.net.InetAddresses.forString; import static java.lang.String.format; +import static java.util.Arrays.asList; import static java.util.Arrays.stream; +import static java.util.Collections.singletonList; import static org.apache.commons.lang.StringUtils.isBlank; import static org.sonar.process.ProcessProperties.CLUSTER_ENABLED; import static org.sonar.process.ProcessProperties.CLUSTER_HOSTS; import static org.sonar.process.ProcessProperties.CLUSTER_NETWORK_INTERFACES; +import static org.sonar.process.ProcessProperties.CLUSTER_NODE_TYPE; import static org.sonar.process.ProcessProperties.CLUSTER_SEARCH_HOSTS; import static org.sonar.process.ProcessProperties.CLUSTER_WEB_LEADER; import static org.sonar.process.ProcessProperties.JDBC_URL; @@ -65,16 +69,22 @@ public class ClusterSettings implements Consumer { throw new MessageException(format("Property [%s] is forbidden", CLUSTER_WEB_LEADER)); } - if (props.valueAsBoolean(ProcessProperties.CLUSTER_ENABLED) && - !props.valueAsBoolean(ProcessProperties.CLUSTER_SEARCH_DISABLED, false) - ) { - ensureMandatoryProperty(props, SEARCH_HOST); - ensureNotLoopback(props, SEARCH_HOST); - } // Mandatory properties + ensureMandatoryProperty(props, CLUSTER_NODE_TYPE); ensureMandatoryProperty(props, CLUSTER_HOSTS); ensureMandatoryProperty(props, CLUSTER_SEARCH_HOSTS); + String nodeTypeValue = props.value(ProcessProperties.CLUSTER_NODE_TYPE); + if (!NodeType.isValid(nodeTypeValue)) { + throw new MessageException(format("Invalid nodeTypeValue for [%s]: [%s], only [%s] are allowed", ProcessProperties.CLUSTER_NODE_TYPE, nodeTypeValue, + Arrays.stream(NodeType.values()).map(NodeType::getValue).collect(Collectors.joining(", ")))); + } + NodeType nodeType = NodeType.parse(nodeTypeValue); + if (nodeType == NodeType.SEARCH) { + ensureMandatoryProperty(props, SEARCH_HOST); + ensureNotLoopback(props, SEARCH_HOST); + } + // H2 Database is not allowed in cluster mode String jdbcUrl = props.value(JDBC_URL); if (isBlank(jdbcUrl) || jdbcUrl.startsWith("jdbc:h2:")) { @@ -162,24 +172,27 @@ public class ClusterSettings implements Consumer { public static List getEnabledProcesses(AppSettings settings) { if (!isClusterEnabled(settings)) { - return Arrays.asList(ProcessId.ELASTICSEARCH, ProcessId.WEB_SERVER, ProcessId.COMPUTE_ENGINE); - } - List enabled = new ArrayList<>(); - if (!settings.getProps().valueAsBoolean(ProcessProperties.CLUSTER_SEARCH_DISABLED)) { - enabled.add(ProcessId.ELASTICSEARCH); + return asList(ProcessId.ELASTICSEARCH, ProcessId.WEB_SERVER, ProcessId.COMPUTE_ENGINE); } - if (!settings.getProps().valueAsBoolean(ProcessProperties.CLUSTER_WEB_DISABLED)) { - enabled.add(ProcessId.WEB_SERVER); - } - - if (!settings.getProps().valueAsBoolean(ProcessProperties.CLUSTER_CE_DISABLED)) { - enabled.add(ProcessId.COMPUTE_ENGINE); + NodeType nodeType = NodeType.parse(settings.getValue(ProcessProperties.CLUSTER_NODE_TYPE).orElse(null)); + switch (nodeType) { + case APPLICATION: + return asList(ProcessId.WEB_SERVER, ProcessId.COMPUTE_ENGINE); + case SEARCH: + return singletonList(ProcessId.ELASTICSEARCH); + default: + throw new IllegalStateException("Unexpected node type "+nodeType); } - return enabled; } public static boolean isLocalElasticsearchEnabled(AppSettings settings) { - return !isClusterEnabled(settings.getProps()) || - !settings.getProps().valueAsBoolean(ProcessProperties.CLUSTER_SEARCH_DISABLED); + + // elasticsearch is enabled on "search" nodes, but disabled on "application" nodes + if (isClusterEnabled(settings.getProps())) { + return NodeType.parse(settings.getValue(ProcessProperties.CLUSTER_NODE_TYPE).orElse(null)) == NodeType.SEARCH; + } + + // elasticsearch is enabled in standalone mode + return true; } } diff --git a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java index f7336820816..7d3c5cee24d 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/SchedulerImplTest.java @@ -94,7 +94,6 @@ public class SchedulerImplTest { @Test public void start_and_stop_sequence_of_ES_WEB_CE_in_order() throws Exception { - enableAllProcesses(); SchedulerImpl underTest = newScheduler(); underTest.schedule(); @@ -132,7 +131,6 @@ public class SchedulerImplTest { } private void enableAllProcesses() { - settings.set(ProcessProperties.CLUSTER_ENABLED, "true"); } @Test @@ -152,7 +150,6 @@ public class SchedulerImplTest { @Test public void all_processes_are_stopped_if_one_process_fails_to_start() throws Exception { - enableAllProcesses(); SchedulerImpl underTest = newScheduler(); processLauncher.makeStartupFail = COMPUTE_ENGINE; @@ -239,59 +236,87 @@ public class SchedulerImplTest { } @Test - public void web_follower_starts_only_when_web_leader_is_operational() throws Exception { + public void search_node_starts_only_elasticsearch() throws Exception { + settings.set(ProcessProperties.CLUSTER_ENABLED, "true"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search"); + SchedulerImpl underTest = newScheduler(); + underTest.schedule(); + + processLauncher.waitForProcessAlive(ProcessId.ELASTICSEARCH); + assertThat(processLauncher.processes).hasSize(1); + + underTest.terminate(); + } + + @Test + public void application_node_starts_only_web_and_ce() throws Exception { + appState.setOperational(ProcessId.ELASTICSEARCH); + settings.set(ProcessProperties.CLUSTER_ENABLED, "true"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application"); + SchedulerImpl underTest = newScheduler(); + underTest.schedule(); + + TestProcess web = processLauncher.waitForProcessAlive(WEB_SERVER); + web.operational = true; + processLauncher.waitForProcessAlive(COMPUTE_ENGINE); + assertThat(processLauncher.processes).hasSize(2); + + underTest.terminate(); + } + + @Test + public void search_node_starts_even_if_web_leader_is_not_yet_operational() throws Exception { // leader takes the lock, so underTest won't get it assertThat(appState.tryToLockWebLeader()).isTrue(); appState.setOperational(ProcessId.ELASTICSEARCH); - enableAllProcesses(); + settings.set(ProcessProperties.CLUSTER_ENABLED, "true"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search"); SchedulerImpl underTest = newScheduler(); underTest.schedule(); processLauncher.waitForProcessAlive(ProcessId.ELASTICSEARCH); assertThat(processLauncher.processes).hasSize(1); - // leader becomes operational -> follower can start - appState.setOperational(ProcessId.WEB_SERVER); - processLauncher.waitForProcessAlive(WEB_SERVER); - underTest.terminate(); } @Test - public void web_server_waits_for_remote_elasticsearch_to_be_started_if_local_es_is_disabled() throws Exception { + public void web_follower_starts_only_when_web_leader_is_operational() throws Exception { + // leader takes the lock, so underTest won't get it + assertThat(appState.tryToLockWebLeader()).isTrue(); + appState.setOperational(ProcessId.ELASTICSEARCH); + settings.set(ProcessProperties.CLUSTER_ENABLED, "true"); - settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application"); SchedulerImpl underTest = newScheduler(); underTest.schedule(); - // WEB and CE wait for ES to be up - assertThat(processLauncher.processes).isEmpty(); + assertThat(processLauncher.processes).hasSize(0); + + // leader becomes operational -> follower can start + appState.setOperational(WEB_SERVER); - // ES becomes operational on another node -> web leader can start - appState.setRemoteOperational(ProcessId.ELASTICSEARCH); processLauncher.waitForProcessAlive(WEB_SERVER); - assertThat(processLauncher.processes).hasSize(1); + processLauncher.waitForProcessAlive(COMPUTE_ENGINE); + assertThat(processLauncher.processes).hasSize(2); underTest.terminate(); } @Test - public void compute_engine_waits_for_remote_elasticsearch_and_web_leader_to_be_started_if_local_es_is_disabled() throws Exception { + public void web_server_waits_for_remote_elasticsearch_to_be_started_if_local_es_is_disabled() throws Exception { settings.set(ProcessProperties.CLUSTER_ENABLED, "true"); - settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true"); - settings.set(ProcessProperties.CLUSTER_WEB_DISABLED, "true"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application"); SchedulerImpl underTest = newScheduler(); underTest.schedule(); - // CE waits for ES and WEB leader to be up + // WEB and CE wait for ES to be up assertThat(processLauncher.processes).isEmpty(); - // ES and WEB leader become operational on another nodes -> CE can start + // ES becomes operational on another node -> web leader can start appState.setRemoteOperational(ProcessId.ELASTICSEARCH); - appState.setRemoteOperational(ProcessId.WEB_SERVER); - - processLauncher.waitForProcessAlive(COMPUTE_ENGINE); + processLauncher.waitForProcessAlive(WEB_SERVER); assertThat(processLauncher.processes).hasSize(1); underTest.terminate(); @@ -303,7 +328,6 @@ public class SchedulerImplTest { } private Scheduler startAll() throws InterruptedException { - enableAllProcesses(); SchedulerImpl scheduler = newScheduler(); scheduler.schedule(); processLauncher.waitForProcess(ELASTICSEARCH).operational = true; diff --git a/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsLoopbackTest.java b/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsLoopbackTest.java index de28db4592c..cd5534fda75 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsLoopbackTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsLoopbackTest.java @@ -37,6 +37,7 @@ import org.sonar.process.MessageException; import static org.sonar.process.ProcessProperties.CLUSTER_ENABLED; import static org.sonar.process.ProcessProperties.CLUSTER_HOSTS; import static org.sonar.process.ProcessProperties.CLUSTER_NETWORK_INTERFACES; +import static org.sonar.process.ProcessProperties.CLUSTER_NODE_TYPE; import static org.sonar.process.ProcessProperties.CLUSTER_SEARCH_HOSTS; import static org.sonar.process.ProcessProperties.JDBC_URL; import static org.sonar.process.ProcessProperties.SEARCH_HOST; @@ -170,6 +171,7 @@ public class ClusterSettingsLoopbackTest { TestAppSettings testAppSettings = new TestAppSettings() .set(CLUSTER_ENABLED, "true") + .set(CLUSTER_NODE_TYPE, "search") .set(CLUSTER_SEARCH_HOSTS, "192.168.233.1, 192.168.233.2,192.168.233.3") .set(CLUSTER_HOSTS, "192.168.233.1, 192.168.233.2,192.168.233.3") .set(SEARCH_HOST, localAddress) diff --git a/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsTest.java index ed89b4685d7..ce6c3a42326 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/config/ClusterSettingsTest.java @@ -33,7 +33,7 @@ import static org.sonar.process.ProcessId.ELASTICSEARCH; import static org.sonar.process.ProcessId.WEB_SERVER; import static org.sonar.process.ProcessProperties.CLUSTER_ENABLED; import static org.sonar.process.ProcessProperties.CLUSTER_HOSTS; -import static org.sonar.process.ProcessProperties.CLUSTER_SEARCH_DISABLED; +import static org.sonar.process.ProcessProperties.CLUSTER_NODE_TYPE; import static org.sonar.process.ProcessProperties.CLUSTER_SEARCH_HOSTS; import static org.sonar.process.ProcessProperties.JDBC_URL; import static org.sonar.process.ProcessProperties.SEARCH_HOST; @@ -67,22 +67,53 @@ public class ClusterSettingsTest { @Test public void getEnabledProcesses_returns_all_processes_by_default() { + settings.set(CLUSTER_ENABLED, "false"); assertThat(ClusterSettings.getEnabledProcesses(settings)).containsOnly(COMPUTE_ENGINE, ELASTICSEARCH, WEB_SERVER); } @Test - public void getEnabledProcesses_returns_all_processes_by_default_in_cluster_mode() { + public void getEnabledProcesses_fails_if_no_node_type_is_set_for_a_cluster_node() { settings.set(CLUSTER_ENABLED, "true"); + settings.set(CLUSTER_NODE_TYPE, ""); - assertThat(ClusterSettings.getEnabledProcesses(settings)).containsOnly(COMPUTE_ENGINE, ELASTICSEARCH, WEB_SERVER); + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Invalid value for [sonar.cluster.node.type]: []"); + + ClusterSettings.getEnabledProcesses(settings); } @Test public void getEnabledProcesses_returns_configured_processes_in_cluster_mode() { settings.set(CLUSTER_ENABLED, "true"); - settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application"); assertThat(ClusterSettings.getEnabledProcesses(settings)).containsOnly(COMPUTE_ENGINE, WEB_SERVER); + + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search"); + + assertThat(ClusterSettings.getEnabledProcesses(settings)).containsOnly(ELASTICSEARCH); + } + + @Test + public void accept_throws_MessageException_if_no_node_type_is_configured() { + settings.set(CLUSTER_ENABLED, "true"); + settings.set(CLUSTER_NODE_TYPE, ""); + + expectedException.expect(MessageException.class); + expectedException.expectMessage("Property [sonar.cluster.node.type] is mandatory"); + + new ClusterSettings().accept(settings.getProps()); + } + + @Test + public void accept_throws_MessageException_if_no_node_type_is_an_illegal_value() { + settings.set(CLUSTER_ENABLED, "true"); + settings.set(CLUSTER_NODE_TYPE, "bla"); + + expectedException.expect(MessageException.class); + expectedException.expectMessage("Invalid nodeTypeValue for [sonar.cluster.node.type]: [bla], only [application, search] are allowed"); + + new ClusterSettings().accept(settings.getProps()); } @Test @@ -99,7 +130,7 @@ public class ClusterSettingsTest { @Test public void accept_throws_MessageException_if_search_enabled_with_loopback() { settings.set(CLUSTER_ENABLED, "true"); - settings.set(CLUSTER_SEARCH_DISABLED, "false"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search"); settings.set(CLUSTER_SEARCH_HOSTS, "192.168.1.1,192.168.1.2"); settings.set(SEARCH_HOST, "::1"); @@ -112,7 +143,7 @@ public class ClusterSettingsTest { @Test public void accept_not_throwing_MessageException_if_search_disabled_with_loopback() { settings.set(CLUSTER_ENABLED, "true"); - settings.set(CLUSTER_SEARCH_DISABLED, "true"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application"); settings.set(CLUSTER_SEARCH_HOSTS, "192.168.1.1,192.168.1.2"); settings.set(SEARCH_HOST, "127.0.0.1"); @@ -155,14 +186,17 @@ public class ClusterSettingsTest { } @Test - public void isLocalElasticsearchEnabled_returns_true_by_default_in_cluster_mode() { + public void isLocalElasticsearchEnabled_returns_true_for_a_search_node() { + settings.set(CLUSTER_ENABLED, "true"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "search"); + assertThat(ClusterSettings.isLocalElasticsearchEnabled(settings)).isTrue(); } @Test - public void isLocalElasticsearchEnabled_returns_false_if_local_es_node_is_disabled_in_cluster_mode() { + public void isLocalElasticsearchEnabled_returns_true_for_a_application_node() { settings.set(CLUSTER_ENABLED, "true"); - settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true"); + settings.set(ProcessProperties.CLUSTER_NODE_TYPE, "application"); assertThat(ClusterSettings.isLocalElasticsearchEnabled(settings)).isFalse(); } @@ -213,6 +247,7 @@ public class ClusterSettingsTest { private static TestAppSettings getClusterSettings() { TestAppSettings testAppSettings = new TestAppSettings() .set(CLUSTER_ENABLED, "true") + .set(CLUSTER_NODE_TYPE, "search") .set(CLUSTER_SEARCH_HOSTS, "localhost") .set(CLUSTER_HOSTS, "192.168.233.1, 192.168.233.2,192.168.233.3") .set(SEARCH_HOST, "192.168.233.1") diff --git a/server/sonar-process/src/main/java/org/sonar/process/NodeType.java b/server/sonar-process/src/main/java/org/sonar/process/NodeType.java new file mode 100644 index 00000000000..55f6e5ef86f --- /dev/null +++ b/server/sonar-process/src/main/java/org/sonar/process/NodeType.java @@ -0,0 +1,53 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.process; + +import javax.annotation.Nullable; + +import static java.util.Arrays.stream; + +public enum NodeType { + APPLICATION("application"), SEARCH("search"); + + private final String value; + + NodeType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static NodeType parse(@Nullable String nodeType) { + if (nodeType == null) { + throw new IllegalStateException("Setting [" + ProcessProperties.CLUSTER_NODE_TYPE + "] is mandatory"); + } + return stream(values()) + .filter(t -> nodeType.equals(t.value)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Invalid value for [" + ProcessProperties.CLUSTER_NODE_TYPE + "]: [" + nodeType + "]")); + } + + public static boolean isValid(String nodeType) { + return stream(values()) + .anyMatch(t -> nodeType.equals(t.value)); + } +} diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java index e9f6d46c0f6..802c4bf27a6 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java +++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java @@ -30,10 +30,8 @@ import java.util.Properties; */ public class ProcessProperties { public static final String CLUSTER_ENABLED = "sonar.cluster.enabled"; - public static final String CLUSTER_CE_DISABLED = "sonar.cluster.ce.disabled"; - public static final String CLUSTER_SEARCH_DISABLED = "sonar.cluster.search.disabled"; + public static final String CLUSTER_NODE_TYPE = "sonar.cluster.node.type"; public static final String CLUSTER_SEARCH_HOSTS = "sonar.cluster.search.hosts"; - public static final String CLUSTER_WEB_DISABLED = "sonar.cluster.web.disabled"; public static final String CLUSTER_HOSTS = "sonar.cluster.hosts"; public static final String CLUSTER_PORT = "sonar.cluster.port"; public static final String CLUSTER_NETWORK_INTERFACES = "sonar.cluster.networkInterfaces"; @@ -138,9 +136,6 @@ public class ProcessProperties { defaults.put(JDBC_TIME_BETWEEN_EVICTION_RUNS_MILLIS, "30000"); defaults.put(CLUSTER_ENABLED, "false"); - defaults.put(CLUSTER_CE_DISABLED, "false"); - defaults.put(CLUSTER_WEB_DISABLED, "false"); - defaults.put(CLUSTER_SEARCH_DISABLED, "false"); defaults.put(CLUSTER_NAME, "sonarqube"); defaults.put(CLUSTER_NETWORK_INTERFACES, ""); defaults.put(CLUSTER_HOSTS, ""); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClientProvider.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClientProvider.java index 20ca3bfe8fb..009d0c4c4ff 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClientProvider.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClientProvider.java @@ -34,8 +34,11 @@ import org.sonar.api.config.Configuration; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; +import org.sonar.process.NodeType; import org.sonar.process.ProcessProperties; +import static org.sonar.process.NodeType.SEARCH; + @ComputeEngineSide @ServerSide public class EsClientProvider extends ProviderAdapter { @@ -46,22 +49,21 @@ public class EsClientProvider extends ProviderAdapter { public EsClient provide(Configuration config) { if (cache == null) { - TransportClient nativeClient; org.elasticsearch.common.settings.Settings.Builder esSettings = org.elasticsearch.common.settings.Settings.builder(); // mandatory property defined by bootstrap process esSettings.put("cluster.name", config.get(ProcessProperties.CLUSTER_NAME).get()); boolean clusterEnabled = config.getBoolean(ProcessProperties.CLUSTER_ENABLED).orElse(false); - if (clusterEnabled && config.getBoolean(ProcessProperties.CLUSTER_SEARCH_DISABLED).orElse(false)) { + boolean searchNode = !clusterEnabled || SEARCH.equals(NodeType.parse(config.get(ProcessProperties.CLUSTER_NODE_TYPE).orElse(null))); + final TransportClient nativeClient = new PreBuiltTransportClient(esSettings.build()); + if (clusterEnabled && !searchNode) { esSettings.put("client.transport.sniff", true); - nativeClient = new PreBuiltTransportClient(esSettings.build()); Arrays.stream(config.getStringArray(ProcessProperties.CLUSTER_SEARCH_HOSTS)) .map(HostAndPort::fromString) .forEach(h -> addHostToClient(h, nativeClient)); LOGGER.info("Connected to remote Elasticsearch: [{}]", displayedAddresses(nativeClient)); } else { - nativeClient = new PreBuiltTransportClient(esSettings.build()); HostAndPort host = HostAndPort.fromParts(config.get(ProcessProperties.SEARCH_HOST).get(), config.getInt(ProcessProperties.SEARCH_PORT).get()); addHostToClient(host, nativeClient); LOGGER.info("Connected to local Elasticsearch: [{}]", displayedAddresses(nativeClient)); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsClientProviderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsClientProviderTest.java index 62f15d002ff..c6ea52d7bcb 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsClientProviderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsClientProviderTest.java @@ -76,7 +76,7 @@ public class EsClientProviderTest { @Test public void connection_to_remote_es_nodes_when_cluster_mode_is_enabled_and_local_es_is_disabled() throws Exception { settings.setProperty(ProcessProperties.CLUSTER_ENABLED, true); - settings.setProperty(ProcessProperties.CLUSTER_SEARCH_DISABLED, true); + settings.setProperty(ProcessProperties.CLUSTER_NODE_TYPE, "application"); settings.setProperty(ProcessProperties.CLUSTER_SEARCH_HOSTS, format("%s:8080,%s:8081", localhost, localhost)); EsClient client = underTest.provide(settings.asConfig()); @@ -97,7 +97,7 @@ public class EsClientProviderTest { @Test public void es_client_provider_must_throw_ISE_when_incorrect_port_is_used_when_search_disabled() throws Exception { settings.setProperty(ProcessProperties.CLUSTER_ENABLED, true); - settings.setProperty(ProcessProperties.CLUSTER_SEARCH_DISABLED, true); + settings.setProperty(ProcessProperties.CLUSTER_NODE_TYPE, "application"); settings.setProperty(ProcessProperties.CLUSTER_SEARCH_HOSTS, format("%s:100000,%s:8081", localhost, localhost)); expectedException.expect(IllegalArgumentException.class); @@ -109,6 +109,7 @@ public class EsClientProviderTest { @Test public void es_client_provider_must_throw_ISE_when_incorrect_port_is_used() throws Exception { settings.setProperty(ProcessProperties.CLUSTER_ENABLED, true); + settings.setProperty(ProcessProperties.CLUSTER_NODE_TYPE, "search"); settings.setProperty(ProcessProperties.SEARCH_HOST, "localhost"); settings.setProperty(ProcessProperties.SEARCH_PORT, "100000"); @@ -121,7 +122,7 @@ public class EsClientProviderTest { @Test public void es_client_provider_must_add_default_port_when_not_specified() throws Exception { settings.setProperty(ProcessProperties.CLUSTER_ENABLED, true); - settings.setProperty(ProcessProperties.CLUSTER_SEARCH_DISABLED, true); + settings.setProperty(ProcessProperties.CLUSTER_NODE_TYPE, "application"); settings.setProperty(ProcessProperties.CLUSTER_SEARCH_HOSTS, format("%s,%s:8081", localhost, localhost)); EsClient client = underTest.provide(settings.asConfig()); diff --git a/tests/src/test/java/org/sonarqube/tests/cluster/Cluster.java b/tests/src/test/java/org/sonarqube/tests/cluster/Cluster.java index d6f0e10c813..3fd8aa7627e 100644 --- a/tests/src/test/java/org/sonarqube/tests/cluster/Cluster.java +++ b/tests/src/test/java/org/sonarqube/tests/cluster/Cluster.java @@ -38,10 +38,10 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; import java.util.function.Predicate; import java.util.stream.Collectors; +import javax.annotation.CheckForNull; -import static org.sonarqube.tests.cluster.Cluster.NodeType.CE; -import static org.sonarqube.tests.cluster.Cluster.NodeType.ES; -import static org.sonarqube.tests.cluster.Cluster.NodeType.WEB; +import static org.sonarqube.tests.cluster.Cluster.NodeType.APPLICATION; +import static org.sonarqube.tests.cluster.Cluster.NodeType.SEARCH; public class Cluster { @@ -65,7 +65,17 @@ public class Cluster { protected static final String CE_JAVA_OPTS = "sonar.ce.javaOpts"; public enum NodeType { - ES, CE, WEB; + SEARCH("search"), APPLICATION("application"); + + final String value; + + NodeType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } public static final EnumSet ALL = EnumSet.allOf(NodeType.class); } @@ -120,10 +130,9 @@ public class Cluster { nodes.stream().forEach( node -> { node.setHzPort(NetworkUtils.getNextAvailablePort(getNonloopbackIPv4Address())); - if (node.getTypes().contains(ES)) { + if (node.getType() == SEARCH) { node.setEsPort(NetworkUtils.getNextAvailablePort(getNonloopbackIPv4Address())); - } - if (node.getTypes().contains(WEB)) { + } else if (node.getType() == APPLICATION) { node.setWebPort(NetworkUtils.getNextAvailablePort(getNonloopbackIPv4Address())); } } @@ -161,29 +170,20 @@ public class Cluster { .map(node -> HostAndPort.fromParts(inet, node.getHzPort()).toString()) .collect(Collectors.joining(",")); String elasticsearchHosts = nodes.stream() - .filter(node -> node.getTypes().contains(ES)) + .filter(node -> node.getType() == SEARCH) .map(node -> HostAndPort.fromParts(inet, node.getEsPort()).toString()) .collect(Collectors.joining(",")); - nodes.stream().forEach( + nodes.forEach( node -> { node.addProperty(CLUSTER_NETWORK_INTERFACES, inet); node.addProperty(CLUSTER_HOSTS, clusterHosts); - node.addProperty(CLUSTER_PORT, Integer.toString(node.getHzPort())); + node.addProperty(CLUSTER_PORT, Integer.toString(node.getHzPort() == null ? -1 : node.getHzPort())); node.addProperty(CLUSTER_SEARCH_HOSTS, elasticsearchHosts); - node.addProperty(SEARCH_PORT, Integer.toString(node.getEsPort())); + node.addProperty(SEARCH_PORT, Integer.toString(node.getEsPort() == null ? -1 : node.getEsPort())); node.addProperty(SEARCH_HOST, inet); - node.addProperty(WEB_PORT, Integer.toString(node.getWebPort())); - - if (!node.getTypes().contains(CE)) { - node.addProperty(CLUSTER_CE_DISABLED, "true"); - } - if (!node.getTypes().contains(WEB)) { - node.addProperty(CLUSTER_WEB_DISABLED, "true"); - } - if (!node.getTypes().contains(ES)) { - node.addProperty(CLUSTER_SEARCH_DISABLED, "true"); - } + node.addProperty(WEB_PORT, Integer.toString(node.getWebPort() == null ? -1 : node.getWebPort())); + node.addProperty(CLUSTER_NODE_TYPE, node.getType().getValue()); } ); } @@ -220,8 +220,8 @@ public class Cluster { return new Cluster(nodes); } - public Builder addNode(EnumSet types) { - nodes.add(new Node(types)); + public Builder addNode(NodeType type) { + nodes.add(new Node(type)); return this; } } @@ -230,15 +230,15 @@ public class Cluster { * A cluster node */ public static class Node { - private final EnumSet types; - private int webPort; - private int esPort; - private int hzPort; + private final NodeType type; + private Integer webPort; + private Integer esPort; + private Integer hzPort; private Orchestrator orchestrator = null; private Properties properties = new Properties(); - public Node(EnumSet types) { - this.types = types; + public Node(NodeType type) { + this.type = type; // Default properties properties.setProperty(CLUSTER_ENABLED, "true"); @@ -277,31 +277,34 @@ public class Cluster { this.orchestrator = orchestrator; } - public EnumSet getTypes() { - return types; + public NodeType getType() { + return type; } - public int getWebPort() { + @CheckForNull + public Integer getWebPort() { return webPort; } - public int getEsPort() { + @CheckForNull + public Integer getEsPort() { return esPort; } - public int getHzPort() { + @CheckForNull + public Integer getHzPort() { return hzPort; } - private void setWebPort(int webPort) { + private void setWebPort(Integer webPort) { this.webPort = webPort; } - private void setEsPort(int esPort) { + private void setEsPort(Integer esPort) { this.esPort = esPort; } - private void setHzPort(int hzPort) { + private void setHzPort(Integer hzPort) { this.hzPort = hzPort; } diff --git a/tests/src/test/java/org/sonarqube/tests/cluster/DataCenterEdition.java b/tests/src/test/java/org/sonarqube/tests/cluster/DataCenterEdition.java index 07007e97c67..b431bcf90bb 100644 --- a/tests/src/test/java/org/sonarqube/tests/cluster/DataCenterEdition.java +++ b/tests/src/test/java/org/sonarqube/tests/cluster/DataCenterEdition.java @@ -22,10 +22,8 @@ package org.sonarqube.tests.cluster; import java.util.concurrent.ExecutionException; -import static java.util.EnumSet.of; -import static org.sonarqube.tests.cluster.Cluster.NodeType.CE; -import static org.sonarqube.tests.cluster.Cluster.NodeType.ES; -import static org.sonarqube.tests.cluster.Cluster.NodeType.WEB; +import static org.sonarqube.tests.cluster.Cluster.NodeType.APPLICATION; +import static org.sonarqube.tests.cluster.Cluster.NodeType.SEARCH; public class DataCenterEdition { @@ -33,11 +31,11 @@ public class DataCenterEdition { public DataCenterEdition() { cluster = Cluster.builder() - .addNode(of(ES)) - .addNode(of(ES)) - .addNode(of(ES)) - .addNode(of(WEB, CE)) - .addNode(of(WEB, CE)) + .addNode(SEARCH) + .addNode(SEARCH) + .addNode(SEARCH) + .addNode(APPLICATION) + .addNode(APPLICATION) .build(); } @@ -48,4 +46,8 @@ public class DataCenterEdition { public void start() throws ExecutionException, InterruptedException { cluster.start(); } + + public Cluster getCluster() { + return cluster; + } } diff --git a/tests/src/test/java/org/sonarqube/tests/cluster/DataCenterEditionTest.java b/tests/src/test/java/org/sonarqube/tests/cluster/DataCenterEditionTest.java index e7b7634fcf3..832d6111c82 100644 --- a/tests/src/test/java/org/sonarqube/tests/cluster/DataCenterEditionTest.java +++ b/tests/src/test/java/org/sonarqube/tests/cluster/DataCenterEditionTest.java @@ -20,14 +20,46 @@ package org.sonarqube.tests.cluster; +import java.io.IOException; +import java.net.InetAddress; +import java.net.ServerSocket; import java.util.concurrent.ExecutionException; +import javax.annotation.Nullable; import org.junit.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.sonarqube.tests.cluster.Cluster.NodeType.APPLICATION; +import static org.sonarqube.tests.cluster.Cluster.NodeType.SEARCH; + public class DataCenterEditionTest { + @Test public void launch() throws ExecutionException, InterruptedException { DataCenterEdition dce = new DataCenterEdition(); + Cluster cluster = dce.getCluster(); dce.start(); + assertThat(cluster.getNodes()) + .extracting(Cluster.Node::getType, n -> isPortBound(false, n.getEsPort()), n -> isPortBound(true, n.getWebPort())) + .containsExactlyInAnyOrder( + tuple(SEARCH, true, false), + tuple(SEARCH, true, false), + tuple(SEARCH, true, false), + tuple(APPLICATION, false, true), + tuple(APPLICATION, false, true) + ); dce.stop(); } + + private static boolean isPortBound(boolean loopback, @Nullable Integer port) { + if (port == null) { + return false; + } + InetAddress inetAddress = loopback ? InetAddress.getLoopbackAddress() : Cluster.getNonloopbackIPv4Address(); + try (ServerSocket socket = new ServerSocket(port, 50, inetAddress)) { + throw new IllegalStateException("A port was set explicitly, but was not bound (port="+port+")"); + } catch (IOException e) { + return true; + } + } } -- 2.39.5