From 1f671d6863b9aad75f9de1264b7f0bb2ba309c7a Mon Sep 17 00:00:00 2001 From: Mark Rekveld Date: Fri, 12 Feb 2021 08:45:18 +0100 Subject: [PATCH] Getting Hazelcast to play nice in Dockerized environment (#3693) [MMF-2168] Hazelcast adjustments and documentation for DCE Docker Images --- .../src/pages/setup/install-cluster.md | 166 +++++++++++++++++- .../src/pages/setup/install-server.md | 2 - .../src/pages/setup/sonar-properties.md | 31 +++- .../sonar/application/AppStateFactory.java | 3 +- .../cluster/ClusterAppStateImplTest.java | 3 +- .../cluster/hz/HazelcastMemberBuilder.java | 45 ++++- .../cluster/hz/InetAdressResolver.java | 34 ++++ .../hz/HazelcastMemberBuilderTest.java | 32 +++- .../cluster/hz/HazelcastMemberImplTest.java | 2 +- 9 files changed, 303 insertions(+), 15 deletions(-) create mode 100644 server/sonar-process/src/main/java/org/sonar/process/cluster/hz/InetAdressResolver.java diff --git a/server/sonar-docs/src/pages/setup/install-cluster.md b/server/sonar-docs/src/pages/setup/install-cluster.md index 7a658a40a3f..5baad811b6c 100644 --- a/server/sonar-docs/src/pages/setup/install-cluster.md +++ b/server/sonar-docs/src/pages/setup/install-cluster.md @@ -56,6 +56,25 @@ SonarSource does not provide specific recommendations for reverse proxy / load b - Ability to balance HTTP requests (load) between the application nodes configured in the SonarQube cluster. - If terminating HTTPS, meets the requirements set out in [Securing SonarQube Behind a Proxy](/setup/operate-server/). - No requirement to preserve or sticky sessions; this is handled by the built-in JWT mechanism. +- Ability to check for node health for routing + +#### Example with HAProxy + +``` +frontend http-in + bind *:80 + bind *:443 ssl crt /etc/ssl/private/ + http-request redirect scheme https unless { ssl_fc } + default_backend sonarqube_server +backend sonarqube_server + balance roundrobin + http-request set-header X-Forwarded-Proto https + option httpchk GET /api/system/status + http-check expect rstring UP|DB_MIGRATION_NEEDED|DB_MIGRATION_RUNNING + default-server check maxconn 200 + server node1 + server node2 +``` ### License @@ -65,7 +84,7 @@ You need a dedicated license to activate the Data Center Edition. If you don't h Don't start this journey alone! As a Data Center Edition subscriber, SonarSource will assist with the setup and configuration of your cluster. Get in touch with [SonarSource Support](https://support.sonarsource.com) for help. -## Configuration +## Installing SonarQube from the ZIP file Additional parameters are required to activate clustering capabilities and specialize each node. These parameters are in addition to standard configuration properties used in a single-node configuration. @@ -155,7 +174,7 @@ sonar.cluster.es.hosts=ip3:9002,ip4:9002,ip5:9002 ... ``` -## Sample Installation Process +### Sample Installation Process The following is an example of the default SonarQube cluster installation process. You need to tailor your installation to the specifics of the target installation environment and the operational requirements of the hosting organization. @@ -187,4 +206,145 @@ The following is an example of the default SonarQube cluster installation proces 4. After all search nodes are running, start all application nodes. 5. Configure the load balancer to proxy with both application nodes. -Congratulations, you have a fully-functional SonarQube cluster. Once you've complete these steps, you can [Operate your Cluster](/setup/operate-cluster/). +## Installing SonarQube from the Docker Image + +You can also install a cluster using our docker images. The general setup is the same but is shifted to a docker specific terminology. + +## Requirements + +### Network + +All containers should be in the same network. This includes search and application nodes. +For the best performance, it is advised to check for low latency between the database and the cluster nodes. + +### Limits + +The limits of each container depend on the workload that each container has. A good starting point would be: + +* cpus: 0.5 +* mem_limit: 4096M +* mem_reservation: 1024M + +The 4Gb mem_limit should not be lower as this is the minimal value for Elasticsearch. + +### Logs + +The sonarqube_logs volume will be populated with a new folder depending on the containers hostname and all logs of this container will be put into this folder. +This behavior also happens when a custom log path is specified via the [Docker Environment Variables](/setup/environment-variables/). + +### Search and Application nodes + +The Application nodes can be scaled using replicas. Please note that this is not the case for the Search nodes: Elasticsearch will not become ready. + +## Example Docker Compose configuration + +[[collapse]] +| ## docker-compose.yml file example +| +| ```yaml +|version: "3" +| +|services: +| sonarqube: +| image: sonarqube:8.6-datacenter-app +| depends_on: +| - db +| - search-1 +| - search-2 +| - search-3 +| networks: +| - sonar-network +| deploy: +| replicas: 2 +| environment: +| SONAR_JDBC_URL: jdbc:postgresql://db:5432/sonar +| SONAR_JDBC_USERNAME: sonar +| SONAR_JDBC_PASSWORD: sonar +| SONAR_WEB_PORT: 9000 +| SONAR_CLUSTER_SEARCH_HOSTS: "search-1,search-2,search-3" +| SONAR_CLUSTER_HOSTS: "sonarqube" +| SONAR_AUTH_JWTBASE64HS256SECRET: "dZ0EB0KxnF++nr5+4vfTCaun/eWbv6gOoXodiAMqcFo=" +| VIRTUAL_HOST: sonarqube.dev.local +| VIRTUAL_PORT: 9000 +| volumes: +| - sonarqube_data:/opt/sonarqube/data +| - sonarqube_extensions:/opt/sonarqube/extensions +| - sonarqube_logs:/opt/sonarqube/logs +| search-1: +| image: sonarqube:8.6-datacenter-search +| hostname: "search-1" +| depends_on: +| - db +| networks: +| - sonar-network +| environment: +| SONAR_JDBC_URL: jdbc:postgresql://db:5432/sonar +| SONAR_JDBC_USERNAME: sonar +| SONAR_JDBC_PASSWORD: sonar +| SONAR_CLUSTER_ES_HOSTS: "search-1,search-2,search-3" +| SONAR_CLUSTER_NODE_NAME: "search-1" +| search-2: +| image: sonarqube:8.6-datacenter-search +| hostname: "search-2" +| depends_on: +| - db +| networks: +| - sonar-network +| environment: +| SONAR_JDBC_URL: jdbc:postgresql://db:5432/sonar +| SONAR_JDBC_USERNAME: sonar +| SONAR_JDBC_PASSWORD: sonar +| SONAR_CLUSTER_ES_HOSTS: "search-1,search-2,search-3" +| SONAR_CLUSTER_NODE_NAME: "search-2" +| search-3: +| image: sonarqube:8.6-datacenter-search +| hostname: "search-3" +| depends_on: +| - db +| networks: +| - sonar-network +| environment: +| SONAR_JDBC_URL: jdbc:postgresql://db:5432/sonar +| SONAR_JDBC_USERNAME: sonar +| SONAR_JDBC_PASSWORD: sonar +| SONAR_CLUSTER_ES_HOSTS: "search-1,search-2,search-3" +| SONAR_CLUSTER_NODE_NAME: "search-3" +| db: +| image: postgres:12 +| networks: +| - sonar-network +| environment: +| POSTGRES_USER: sonar +| POSTGRES_PASSWORD: sonar +| volumes: +| - postgresql:/var/lib/postgresql +| - postgresql_data:/var/lib/postgresql/data +| proxy: +| image: jwilder/nginx-proxy +| ports: +| - "80:80" +| volumes: +| - /var/run/docker.sock:/tmp/docker.sock:ro +| networks: +| - sonar-network +| - sonar-public +| +|networks: +| sonar-network: +| ipam: +| driver: default +| config: +| - subnet: 172.28.2.0/24 +| sonar-public: +| driver: bridge +| +|volumes: +| sonarqube_data: +| sonarqube_extensions: +| sonarqube_logs: +| postgresql: +| postgresql_data: +| ``` + +## Next Steps +Once you've complete these steps, check out the [Operate your Cluster](/setup/operate-cluster/) documentation. diff --git a/server/sonar-docs/src/pages/setup/install-server.md b/server/sonar-docs/src/pages/setup/install-server.md index c6375a80b0e..441d0bf8422 100644 --- a/server/sonar-docs/src/pages/setup/install-server.md +++ b/server/sonar-docs/src/pages/setup/install-server.md @@ -249,7 +249,6 @@ If you're using [Docker Compose](https://docs.docker.com/compose/), use the foll | - sonarqube_data:/opt/sonarqube/data | - sonarqube_extensions:/opt/sonarqube/extensions | - sonarqube_logs:/opt/sonarqube/logs -| - sonarqube_temp:/opt/sonarqube/temp | ports: | - "9000:9000" | db: @@ -265,7 +264,6 @@ If you're using [Docker Compose](https://docs.docker.com/compose/), use the foll | sonarqube_data: | sonarqube_extensions: | sonarqube_logs: -| sonarqube_temp: | postgresql: | postgresql_data: | ``` diff --git a/server/sonar-docs/src/pages/setup/sonar-properties.md b/server/sonar-docs/src/pages/setup/sonar-properties.md index 860eba21b22..adf1b5ef2a3 100644 --- a/server/sonar-docs/src/pages/setup/sonar-properties.md +++ b/server/sonar-docs/src/pages/setup/sonar-properties.md @@ -341,6 +341,36 @@ If SonarQube is behind a reverse proxy, then the following value allows to displ Default value (which was "combined" before version 6.2) is equivalent to "combined + SQ HTTP request ID": `SONAR_WEB_ACCESSLOGS_PATTERN=%h %l %u [%t] "%r" %s %b "%i{Referer}" "%i{User-Agent}" "%reqAttribute{ID}"` +## DataCenter Edition + +**`SONAR_CLUSTER_NAME=sonarqube`** + +The name of the cluster. Required if multiple clusters are present on the same network. For example, this prevents mixing Production and Preproduction clusters. This will be the name stored in the Hazelcast cluster and used as the name of the Elasticsearch cluster. + +**`SONAR_CLUSTER_SEARCH_HOSTS`** + +Comma-delimited list of search hosts in the cluster. The list can contain either the host or the host and port, but not both. The item format is `ip/hostname` for host only or`ip/hostname:port` for host and port. `ip/hostname` can also be set to the service name of the search containers . + +### Search Nodes Only + +**`SONAR_CLUSTER_ES_HOSTS`** + +Comma-delimited list of search hosts in the cluster. The list can contain either the host or the host and port but not both. The item format is `ip/hostname` for host only or`ip/hostname:port` for host and port, while `ip/hostname` can also be set to the service name of the search containers. + +**`SONAR_CLUSTER_NODE_NAME`** + +The name of the node that is used on Elasticsearch and stored in Hazelcast member attribute (NODE_NAME) + +### Application Nodes Only + +**`SONAR_CLUSTER_HOSTS`** + +Comma-delimited list of all **application** hosts in the cluster. This value must contain **only application hosts**. Each item in the list must contain the port if the default `SONAR_CLUSTER_NODE_PORT` value is not used. Item format is `ip/hostname`, `ip/hostname:port`. `ip/hostname` can also be set to the service name of the application containers. + +**`SONAR_CLUSTER_NODE_PORT`** + +The Hazelcast port for communication with each application member of the cluster. Default: `9003` + ## Others **`SONAR_NOTIFICATIONS_DELAY=60`** @@ -359,4 +389,3 @@ Telemetry - Share anonymous SonarQube statistics. By sharing anonymous SonarQube **`SONAR_SEARCH_HTTPPORT=-1`** Elasticsearch HTTP connector - diff --git a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java index bb445314b6c..28c2e0001a4 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java +++ b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java @@ -33,6 +33,7 @@ import org.sonar.process.ProcessId; import org.sonar.process.Props; import org.sonar.process.cluster.hz.HazelcastMember; import org.sonar.process.cluster.hz.HazelcastMemberBuilder; +import org.sonar.process.cluster.hz.InetAdressResolver; import static java.util.Arrays.asList; import static org.sonar.process.ProcessProperties.Property.CLUSTER_HZ_HOSTS; @@ -59,7 +60,7 @@ public class AppStateFactory { } private static HazelcastMember createHzMember(Props props) { - HazelcastMemberBuilder builder = new HazelcastMemberBuilder() + HazelcastMemberBuilder builder = new HazelcastMemberBuilder(new InetAdressResolver()) .setNetworkInterface(props.nonNullValue(CLUSTER_NODE_HOST.getKey())) .setMembers(asList(props.nonNullValue(CLUSTER_HZ_HOSTS.getKey()).split(","))) .setNodeName(props.nonNullValue(CLUSTER_NODE_NAME.getKey())) diff --git a/server/sonar-main/src/test/java/org/sonar/application/cluster/ClusterAppStateImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/cluster/ClusterAppStateImplTest.java index 257e72ccd1f..eef179ed076 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/cluster/ClusterAppStateImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/cluster/ClusterAppStateImplTest.java @@ -34,6 +34,7 @@ import org.sonar.process.NetworkUtilsImpl; import org.sonar.process.ProcessId; import org.sonar.process.cluster.hz.HazelcastMember; import org.sonar.process.cluster.hz.HazelcastMemberBuilder; +import org.sonar.process.cluster.hz.InetAdressResolver; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -152,7 +153,7 @@ public class ClusterAppStateImplTest { // use loopback for support of offline builds InetAddress loopback = InetAddress.getLoopbackAddress(); - return new HazelcastMemberBuilder() + return new HazelcastMemberBuilder(new InetAdressResolver()) .setProcessId(ProcessId.COMPUTE_ENGINE) .setNodeName("bar") .setPort(NetworkUtilsImpl.INSTANCE.getNextLoopbackAvailablePort()) diff --git a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberBuilder.java b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberBuilder.java index a40943a970b..e5bc3e8cec6 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberBuilder.java +++ b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberBuilder.java @@ -24,11 +24,15 @@ import com.hazelcast.config.JoinConfig; import com.hazelcast.config.MemberAttributeConfig; import com.hazelcast.config.NetworkConfig; import com.hazelcast.core.Hazelcast; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import javax.annotation.CheckForNull; + +import com.hazelcast.util.AddressUtil; import org.sonar.process.ProcessId; import org.sonar.process.cluster.hz.HazelcastMember.Attribute; @@ -36,14 +40,22 @@ import static java.lang.String.format; import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HZ_PORT; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; public class HazelcastMemberBuilder { + private static final Logger LOG = Loggers.get(HazelcastMemberBuilder.class); private String nodeName; private int port; private ProcessId processId; private String networkInterface; private List members = new ArrayList<>(); + private final InetAdressResolver inetAdressResolver; + + public HazelcastMemberBuilder(InetAdressResolver inetAdressResolver) { + this.inetAdressResolver = inetAdressResolver; + } public HazelcastMemberBuilder setNodeName(String s) { this.nodeName = s; @@ -78,12 +90,39 @@ public class HazelcastMemberBuilder { * port is automatically added. */ public HazelcastMemberBuilder setMembers(Collection c) { - this.members = c.stream() - .map(host -> host.contains(":") ? host : format("%s:%s", host, CLUSTER_NODE_HZ_PORT.getDefaultValue())) - .collect(Collectors.toList()); + this.members.addAll(c.stream().map(this::extractMembers).flatMap(Collection::stream).collect(Collectors.toList())); return this; } + private List extractMembers(String host) { + LOG.debug("Trying to add host: " + host); + String hostStripped = host.split(":")[0]; + if (AddressUtil.isIpAddress(hostStripped)) { + LOG.debug("Found ip based host config for host: " + host); + return Collections.singletonList(host.contains(":") ? host : format("%s:%s", host, CLUSTER_NODE_HZ_PORT.getDefaultValue())); + } else { + List membersToAdd = new ArrayList<>(); + for (String memberIp : getAllByName(hostStripped)){ + String prefix = memberIp.split("/")[1]; + LOG.debug("Found IP for: " + hostStripped + " : " + prefix); + String memberPort = host.contains(":") ? host.split(":")[1] : CLUSTER_NODE_HZ_PORT.getDefaultValue(); + String member = prefix + ":" + memberPort; + membersToAdd.add(member); + } + return membersToAdd; + } + } + + List getAllByName(String hostname) { + LOG.debug("Trying to resolve Hostname: " + hostname); + try { + return inetAdressResolver.getAllByName(hostname); + } catch (UnknownHostException e) { + LOG.error("Host could not be found\n" + e.getMessage()); + } + return new ArrayList<>(); + } + public HazelcastMember build() { Config config = new Config(); // do not use the value defined by property sonar.cluster.name. diff --git a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/InetAdressResolver.java b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/InetAdressResolver.java new file mode 100644 index 00000000000..3211a94ca9f --- /dev/null +++ b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/InetAdressResolver.java @@ -0,0 +1,34 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.process.cluster.hz; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class InetAdressResolver { + + public List getAllByName(String hostname) throws UnknownHostException { + return Arrays.stream(InetAddress.getAllByName(hostname)).map(InetAddress::toString).collect(Collectors.toList()); + } + +} diff --git a/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberBuilderTest.java b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberBuilderTest.java index fa3d86788f4..2fc1df20d3c 100644 --- a/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberBuilderTest.java +++ b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberBuilderTest.java @@ -20,6 +20,12 @@ package org.sonar.process.cluster.hz; import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -31,6 +37,8 @@ import org.sonar.process.ProcessId; import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HZ_PORT; public class HazelcastMemberBuilderTest { @@ -42,7 +50,25 @@ public class HazelcastMemberBuilderTest { // use loopback for support of offline builds private InetAddress loopback = InetAddress.getLoopbackAddress(); - private HazelcastMemberBuilder underTest = new HazelcastMemberBuilder(); + private InetAdressResolver inetAdressResolver = mock(InetAdressResolver.class); + private HazelcastMemberBuilder underTest = new HazelcastMemberBuilder(inetAdressResolver); + + @Before + public void before() throws UnknownHostException { + when(inetAdressResolver.getAllByName("foo")).thenReturn(Collections.singletonList("foo/5.6.7.8")); + when(inetAdressResolver.getAllByName("bar")).thenReturn(Collections.singletonList("bar/8.7.6.5")); + when(inetAdressResolver.getAllByName("wizz")).thenReturn(Arrays.asList("wizz/1.2.3.4", "wizz/2.3.4.5", "wizz/3.4.5.6")); + when(inetAdressResolver.getAllByName("ninja")).thenReturn(Arrays.asList("ninja/4.5.6.7", "ninja/5.6.7.8")); + } + + @Test + public void testMultipleIPsByHostname() { + underTest.setMembers(asList("wizz:9001", "ninja")); + + List members = underTest.getMembers(); + assertThat(members).containsExactlyInAnyOrder("1.2.3.4:9001", "2.3.4.5:9001", "3.4.5.6:9001", "4.5.6.7:9003", "5.6.7.8:9003"); + + } @Test public void build_member() { @@ -70,8 +96,8 @@ public class HazelcastMemberBuilderTest { underTest.setMembers(asList("foo", "bar:9100", "1.2.3.4")); assertThat(underTest.getMembers()).containsExactly( - "foo:" + CLUSTER_NODE_HZ_PORT.getDefaultValue(), - "bar:9100", + "5.6.7.8:" + CLUSTER_NODE_HZ_PORT.getDefaultValue(), + "8.7.6.5:9100", "1.2.3.4:" + CLUSTER_NODE_HZ_PORT.getDefaultValue()); } diff --git a/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberImplTest.java b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberImplTest.java index 4f326d16597..11caec23c0f 100644 --- a/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberImplTest.java +++ b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberImplTest.java @@ -110,7 +110,7 @@ public class HazelcastMemberImplTest { } private static HazelcastMember newHzMember(int port, int... otherPorts) { - return new HazelcastMemberBuilder() + return new HazelcastMemberBuilder(new InetAdressResolver()) .setProcessId(ProcessId.COMPUTE_ENGINE) .setNodeName("name" + port) .setPort(port) -- 2.39.5