Browse Source

SONAR-15230 Enable DNS lookup in k8s for HZ

tags/9.1.0.47736
Jacek 2 years ago
parent
commit
78a0a38cd3

+ 1
- 0
build.gradle View File

@@ -332,6 +332,7 @@ subprojects {
dependencySet(group: 'com.hazelcast', version: '4.2') {
entry 'hazelcast'
}
dependency 'com.hazelcast:hazelcast-kubernetes:2.2.3'
dependency 'com.ibm.icu:icu4j:3.4.4'
dependency 'com.microsoft.sqlserver:mssql-jdbc:9.2.0.jre11'
dependency 'com.oracle.database.jdbc:ojdbc8:19.3.0.0'

+ 1
- 0
server/sonar-ce/build.gradle View File

@@ -11,6 +11,7 @@ dependencies {
compile 'com.google.guava:guava'
compile 'com.google.protobuf:protobuf-java'
compile 'com.hazelcast:hazelcast'
compile 'com.hazelcast:hazelcast-kubernetes'
compile 'commons-io:commons-io'
compile 'org.apache.commons:commons-dbcp2'
compile 'org.nanohttpd:nanohttpd'

+ 1
- 1
server/sonar-docs/src/pages/setup/upgrade-notes.md View File

@@ -44,4 +44,4 @@ SonarQube 8.9 supports the following database versions:
**Webhooks aren't allowed to target the instance**
To improve security, webhooks, by default, aren't allowed to point to the SonarQube server. You can change this behavior in the configuration. ([SONAR-14682](https://jira.sonarsource.com/browse/SONAR-14682)).

[Full release notes](https://jira.sonarsource.com/secure/ReleaseNote.jspa?projectId=10930&version=16710)
[Full release notes](https://jira.sonarsource.com/secure/ReleaseNote.jspa?projectId=10930&version=16710)

+ 5
- 2
server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java View File

@@ -33,15 +33,17 @@ import org.sonar.process.ProcessId;
import org.sonar.process.Props;
import org.sonar.process.cluster.hz.HazelcastMember;
import org.sonar.process.cluster.hz.HazelcastMemberBuilder;
import org.sonar.process.cluster.hz.InetAdressResolver;

import static java.util.Arrays.asList;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_HZ_HOSTS;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_KUBERNETES;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HOST;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HZ_PORT;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_NAME;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_HOSTS;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_PASSWORD;
import static org.sonar.process.cluster.hz.JoinConfigurationType.KUBERNETES;
import static org.sonar.process.cluster.hz.JoinConfigurationType.TCP_IP;

public class AppStateFactory {
private final AppSettings settings;
@@ -61,7 +63,8 @@ public class AppStateFactory {
}

private static HazelcastMember createHzMember(Props props) {
HazelcastMemberBuilder builder = new HazelcastMemberBuilder(new InetAdressResolver())
boolean isRunningOnKubernetes = props.valueAsBoolean(CLUSTER_KUBERNETES.getKey(), Boolean.parseBoolean(CLUSTER_KUBERNETES.getDefaultValue()));
HazelcastMemberBuilder builder = new HazelcastMemberBuilder(isRunningOnKubernetes ? KUBERNETES : TCP_IP)
.setNetworkInterface(props.nonNullValue(CLUSTER_NODE_HOST.getKey()))
.setMembers(asList(props.nonNullValue(CLUSTER_HZ_HOSTS.getKey()).split(",")))
.setNodeName(props.nonNullValue(CLUSTER_NODE_NAME.getKey()))

+ 2
- 2
server/sonar-main/src/test/java/org/sonar/application/cluster/ClusterAppStateImplTest.java View File

@@ -35,7 +35,7 @@ import org.sonar.process.NetworkUtilsImpl;
import org.sonar.process.ProcessId;
import org.sonar.process.cluster.hz.HazelcastMember;
import org.sonar.process.cluster.hz.HazelcastMemberBuilder;
import org.sonar.process.cluster.hz.InetAdressResolver;
import org.sonar.process.cluster.hz.JoinConfigurationType;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
@@ -171,7 +171,7 @@ public class ClusterAppStateImplTest {
// use loopback for support of offline builds
InetAddress loopback = InetAddress.getLoopbackAddress();

return new HazelcastMemberBuilder(new InetAdressResolver())
return new HazelcastMemberBuilder(JoinConfigurationType.TCP_IP)
.setProcessId(ProcessId.COMPUTE_ENGINE)
.setNodeName("bar")
.setPort(NetworkUtilsImpl.INSTANCE.getNextLoopbackAvailablePort())

+ 1
- 0
server/sonar-process/build.gradle View File

@@ -16,6 +16,7 @@ dependencies {
compile 'com.google.guava:guava'
compile 'com.google.protobuf:protobuf-java'
compile 'com.hazelcast:hazelcast'
compile 'com.hazelcast:hazelcast-kubernetes'
compile 'org.slf4j:jul-to-slf4j'
compile 'org.slf4j:slf4j-api'
compile project(':sonar-core')

+ 1
- 0
server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java View File

@@ -114,6 +114,7 @@ public class ProcessProperties {
SOCKS_PROXY_PORT("socksProxyPort"),

CLUSTER_ENABLED("sonar.cluster.enabled", "false"),
CLUSTER_KUBERNETES("sonar.cluster.kubernetes", "false"),
CLUSTER_NODE_TYPE("sonar.cluster.node.type"),
CLUSTER_SEARCH_HOSTS("sonar.cluster.search.hosts"),
CLUSTER_HZ_HOSTS("sonar.cluster.hosts"),

+ 22
- 56
server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberBuilder.java View File

@@ -24,36 +24,29 @@ import com.hazelcast.config.JoinConfig;
import com.hazelcast.config.MemberAttributeConfig;
import com.hazelcast.config.NetworkConfig;
import com.hazelcast.core.Hazelcast;
import com.hazelcast.internal.util.AddressUtil;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import javax.annotation.CheckForNull;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.process.ProcessId;
import org.sonar.process.cluster.hz.HazelcastMember.Attribute;

import static java.lang.String.format;
import static java.util.Collections.singletonList;
import static java.util.Objects.requireNonNull;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HZ_PORT;
import static org.sonar.process.cluster.hz.JoinConfigurationType.KUBERNETES;
import static org.sonar.process.cluster.hz.JoinConfigurationType.TCP_IP;

public class HazelcastMemberBuilder {

private static final Logger LOG = Loggers.get(HazelcastMemberBuilder.class);
private String nodeName;
private int port;
private ProcessId processId;
private String networkInterface;
private List<String> members = new ArrayList<>();
private final InetAdressResolver inetAdressResolver;
private final MembersResolver membersResolver;
private final List<String> members = new ArrayList<>();
private final JoinConfigurationType type;

public HazelcastMemberBuilder(InetAdressResolver inetAdressResolver) {
this.inetAdressResolver = inetAdressResolver;
public HazelcastMemberBuilder(JoinConfigurationType type) {
this.type = type;
this.membersResolver = TCP_IP.equals(type) ? new TcpIpMembersResolver() : new NopMembersResolver();
}

public HazelcastMemberBuilder setNodeName(String s) {
@@ -79,49 +72,14 @@ public class HazelcastMemberBuilder {
return this;
}

@CheckForNull
List<String> getMembers() {
return members;
}

/**
* Adds references to cluster members. If port is missing, then default
* port is automatically added.
* Adds references to cluster members
*/
public HazelcastMemberBuilder setMembers(Collection<String> c) {
this.members.addAll(c.stream().map(this::extractMembers).flatMap(Collection::stream).collect(Collectors.toList()));
public HazelcastMemberBuilder setMembers(Collection<String> members) {
this.members.addAll(members);
return this;
}

private List<String> extractMembers(String host) {
LOG.debug("Trying to add host: " + host);
String hostStripped = host.split(":")[0];
if (AddressUtil.isIpAddress(hostStripped)) {
LOG.debug("Found ip based host config for host: " + host);
return Collections.singletonList(host.contains(":") ? host : format("%s:%s", host, CLUSTER_NODE_HZ_PORT.getDefaultValue()));
} else {
List<String> membersToAdd = new ArrayList<>();
for (String memberIp : getAllByName(hostStripped)) {
String prefix = memberIp.split("/")[1];
LOG.debug("Found IP for: " + hostStripped + " : " + prefix);
String memberPort = host.contains(":") ? host.split(":")[1] : CLUSTER_NODE_HZ_PORT.getDefaultValue();
String member = prefix + ":" + memberPort;
membersToAdd.add(member);
}
return membersToAdd;
}
}

List<String> getAllByName(String hostname) {
LOG.debug("Trying to resolve Hostname: " + hostname);
try {
return inetAdressResolver.getAllByName(hostname);
} catch (UnknownHostException e) {
LOG.error("Host could not be found\n" + e.getMessage());
}
return new ArrayList<>();
}

public HazelcastMember build() {
Config config = new Config();
// do not use the value defined by property sonar.cluster.name.
@@ -140,12 +98,20 @@ public class HazelcastMemberBuilder {
.setEnabled(true)
.setInterfaces(singletonList(requireNonNull(networkInterface, "Network interface is missing")));

// Only allowing TCP/IP configuration
JoinConfig joinConfig = netConfig.getJoin();
joinConfig.getAwsConfig().setEnabled(false);
joinConfig.getMulticastConfig().setEnabled(false);
joinConfig.getTcpIpConfig().setEnabled(true);
joinConfig.getTcpIpConfig().setMembers(requireNonNull(members, "Members are missing"));

List<String> resolvedNodes = membersResolver.resolveMembers(this.members);
if (KUBERNETES.equals(type)) {
joinConfig.getKubernetesConfig().setEnabled(true)
.setProperty("service-dns", requireNonNull(resolvedNodes.get(0), "Service DNS is missing"))
.setProperty("service-port", "9003");
} else {
joinConfig.getTcpIpConfig().setEnabled(true);
joinConfig.getTcpIpConfig().setMembers(requireNonNull(resolvedNodes, "Members are missing"));
}

// We are not using the partition group of Hazelcast, so disabling it
config.getPartitionGroupConfig().setEnabled(false);


+ 25
- 0
server/sonar-process/src/main/java/org/sonar/process/cluster/hz/JoinConfigurationType.java View File

@@ -0,0 +1,25 @@
/*
* SonarQube
* Copyright (C) 2009-2021 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.process.cluster.hz;

public enum JoinConfigurationType {
TCP_IP,
KUBERNETES
}

+ 26
- 0
server/sonar-process/src/main/java/org/sonar/process/cluster/hz/MembersResolver.java View File

@@ -0,0 +1,26 @@
/*
* SonarQube
* Copyright (C) 2009-2021 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.process.cluster.hz;

import java.util.List;

interface MembersResolver {
List<String> resolveMembers(List<String> membersToResolve);
}

server/sonar-process/src/main/java/org/sonar/process/cluster/hz/InetAdressResolver.java → server/sonar-process/src/main/java/org/sonar/process/cluster/hz/NopMembersResolver.java View File

@@ -19,16 +19,12 @@
*/
package org.sonar.process.cluster.hz;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;

public class InetAdressResolver {
public List<String> getAllByName(String hostname) throws UnknownHostException {
return Arrays.stream(InetAddress.getAllByName(hostname)).map(InetAddress::toString).collect(Collectors.toList());
class NopMembersResolver implements MembersResolver {
@Override
public List<String> resolveMembers(List<String> membersToResolve) {
return membersToResolve;
}

}

+ 74
- 0
server/sonar-process/src/main/java/org/sonar/process/cluster/hz/TcpIpMembersResolver.java View File

@@ -0,0 +1,74 @@
/*
* SonarQube
* Copyright (C) 2009-2021 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.process.cluster.hz;

import com.hazelcast.internal.util.AddressUtil;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;

import static java.lang.String.format;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HZ_PORT;

class TcpIpMembersResolver implements MembersResolver {
private static final Logger LOG = Loggers.get(TcpIpMembersResolver.class);

@Override
public List<String> resolveMembers(List<String> membersToResolve) {
return membersToResolve.stream().map(this::extractMembers).flatMap(Collection::stream).collect(Collectors.toList());
}

private List<String> extractMembers(String host) {
LOG.debug("Trying to add host: " + host);
String hostStripped = host.split(":")[0];
if (AddressUtil.isIpAddress(hostStripped)) {
LOG.debug("Found ip based host config for host: " + host);
return Collections.singletonList(host.contains(":") ? host : format("%s:%s", host, CLUSTER_NODE_HZ_PORT.getDefaultValue()));
} else {
List<String> membersToAdd = new ArrayList<>();
for (String memberIp : getAllByName(hostStripped)) {
String prefix = memberIp.split("/")[1];
LOG.debug("Found IP for: " + hostStripped + " : " + prefix);
String memberPort = host.contains(":") ? host.split(":")[1] : CLUSTER_NODE_HZ_PORT.getDefaultValue();
String member = prefix + ":" + memberPort;
membersToAdd.add(member);
}
return membersToAdd;
}
}

private List<String> getAllByName(String hostname) {
LOG.debug("Trying to resolve Hostname: " + hostname);
try {
return Arrays.stream(InetAddress.getAllByName(hostname)).map(InetAddress::toString).collect(Collectors.toList());
} catch (UnknownHostException e) {
LOG.error("Host could not be found: " + e.getMessage());
}
return new ArrayList<>();
}

}

+ 46
- 40
server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberBuilderTest.java View File

@@ -20,58 +20,74 @@
package org.sonar.process.cluster.hz;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.DisableOnDebug;
import org.junit.rules.ExpectedException;
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
import org.sonar.process.NetworkUtilsImpl;
import org.sonar.process.ProcessId;

import static java.util.Arrays.asList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HZ_PORT;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

public class HazelcastMemberBuilderTest {

@Rule
public ExpectedException expectedException = ExpectedException.none();
@Rule
public TestRule safeguardTimeout = new DisableOnDebug(Timeout.seconds(60));

// use loopback for support of offline builds
private final InetAddress loopback = InetAddress.getLoopbackAddress();
private final InetAdressResolver inetAdressResolver = mock(InetAdressResolver.class);
private final HazelcastMemberBuilder underTest = new HazelcastMemberBuilder(inetAdressResolver);

@Before
public void before() throws UnknownHostException {
when(inetAdressResolver.getAllByName("foo")).thenReturn(Collections.singletonList("foo/5.6.7.8"));
when(inetAdressResolver.getAllByName("bar")).thenReturn(Collections.singletonList("bar/8.7.6.5"));
when(inetAdressResolver.getAllByName("wizz")).thenReturn(Arrays.asList("wizz/1.2.3.4", "wizz/2.3.4.5", "wizz/3.4.5.6"));
when(inetAdressResolver.getAllByName("ninja")).thenReturn(Arrays.asList("ninja/4.5.6.7", "ninja/5.6.7.8"));

@Test
public void build_tcp_ip_member_hostaddress() {
HazelcastMember member = new HazelcastMemberBuilder(JoinConfigurationType.TCP_IP)
.setMembers(Collections.singletonList(loopback.getHostAddress()))
.setProcessId(ProcessId.COMPUTE_ENGINE)
.setNodeName("bar")
.setPort(NetworkUtilsImpl.INSTANCE.getNextLoopbackAvailablePort())
.setNetworkInterface(loopback.getHostAddress())
.build();

assertThat(member.getUuid()).isNotNull();
assertThat(member.getClusterTime()).isPositive();
assertThat(member.getCluster().getMembers()).hasSize(1);
assertThat(member.getMemberUuids()).containsOnlyOnce(member.getUuid());

assertThat(member.getAtomicReference("baz")).isNotNull();
assertThat(member.getLock("baz")).isNotNull();
assertThat(member.getReplicatedMap("baz")).isNotNull();

member.close();
}

@Test
public void testMultipleIPsByHostname() {
underTest.setMembers(asList("wizz:9001", "ninja"));
public void build_tcp_ip_member_hostname() {
HazelcastMember member = new HazelcastMemberBuilder(JoinConfigurationType.TCP_IP)
.setMembers(Collections.singletonList(loopback.getHostName()))
.setProcessId(ProcessId.COMPUTE_ENGINE)
.setNodeName("bar")
.setPort(NetworkUtilsImpl.INSTANCE.getNextLoopbackAvailablePort())
.setNetworkInterface(loopback.getHostAddress())
.build();

assertThat(member.getUuid()).isNotNull();
assertThat(member.getClusterTime()).isPositive();
assertThat(member.getCluster().getMembers()).hasSize(1);
assertThat(member.getMemberUuids()).containsOnlyOnce(member.getUuid());

List<String> members = underTest.getMembers();
assertThat(members).containsExactlyInAnyOrder("1.2.3.4:9001", "2.3.4.5:9001", "3.4.5.6:9001", "4.5.6.7:9003", "5.6.7.8:9003");
assertThat(member.getAtomicReference("baz")).isNotNull();
assertThat(member.getLock("baz")).isNotNull();
assertThat(member.getReplicatedMap("baz")).isNotNull();

member.close();
}

@Test
public void build_member() {
HazelcastMember member = underTest
public void build_kubernetes_member() {
HazelcastMember member = new HazelcastMemberBuilder(JoinConfigurationType.KUBERNETES)
.setMembers(Collections.singletonList(loopback.getHostAddress()))
.setProcessId(ProcessId.COMPUTE_ENGINE)
.setNodeName("bar")
.setPort(NetworkUtilsImpl.INSTANCE.getNextLoopbackAvailablePort())
@@ -90,21 +106,11 @@ public class HazelcastMemberBuilderTest {
member.close();
}

@Test
public void default_port_is_added_when_missing() {
underTest.setMembers(asList("foo", "bar:9100", "1.2.3.4"));

assertThat(underTest.getMembers()).containsExactly(
"5.6.7.8:" + CLUSTER_NODE_HZ_PORT.getDefaultValue(),
"8.7.6.5:9100",
"1.2.3.4:" + CLUSTER_NODE_HZ_PORT.getDefaultValue());
}

@Test
public void fail_if_elasticsearch_process() {
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Hazelcast must not be enabled on Elasticsearch node");
underTest.setProcessId(ProcessId.ELASTICSEARCH);
var builder = new HazelcastMemberBuilder(JoinConfigurationType.TCP_IP);
assertThatThrownBy(() -> builder.setProcessId(ProcessId.ELASTICSEARCH))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Hazelcast must not be enabled on Elasticsearch node");
}
}

+ 1
- 1
server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberImplTest.java View File

@@ -110,7 +110,7 @@ public class HazelcastMemberImplTest {
}

private static HazelcastMember newHzMember(int port, int... otherPorts) {
return new HazelcastMemberBuilder(new InetAdressResolver())
return new HazelcastMemberBuilder(JoinConfigurationType.TCP_IP)
.setProcessId(ProcessId.COMPUTE_ENGINE)
.setNodeName("name" + port)
.setPort(port)

Loading…
Cancel
Save