class StartupBarrierFactory {
public StartupBarrier create(ProcessEntryPoint entryPoint) {
- if (entryPoint.getProps().valueAsBoolean(ProcessProperties.WEB_DISABLED)) {
+ if (entryPoint.getProps().valueAsBoolean(ProcessProperties.CLUSTER_WEB_DISABLED)) {
return () -> true;
}
return new WebServerBarrier(entryPoint.getSharedDir());
@Test
public void do_not_wait_for_web_server_if_it_is_disabled() {
- props.set(ProcessProperties.WEB_DISABLED, "true");
+ props.set(ProcessProperties.CLUSTER_WEB_DISABLED, "true");
StartupBarrier barrier = underTest.create(entryPoint);
assertThat(barrier).isNotInstanceOf(WebServerBarrier.class);
import org.sonar.db.DbTester;
import org.sonar.db.property.PropertyDto;
import org.sonar.process.ProcessId;
+import org.sonar.process.ProcessProperties;
import org.sonar.process.Props;
import static java.lang.String.valueOf;
@Test
public void real_start() throws IOException {
- Properties properties = new Properties();
+ Properties properties = ProcessProperties.defaults();
File homeDir = tempFolder.newFolder();
File dataDir = new File(homeDir, "data");
File tmpDir = new File(homeDir, "tmp");
properties.setProperty(PATH_TEMP, tmpDir.getAbsolutePath());
properties.setProperty(PROPERTY_PROCESS_INDEX, valueOf(ProcessId.COMPUTE_ENGINE.getIpcIndex()));
properties.setProperty(PROPERTY_SHARED_PATH, tmpDir.getAbsolutePath());
- String url = ((BasicDataSource) dbTester.database().getDataSource()).getUrl();
- properties.setProperty(DatabaseProperties.PROP_URL, url);
+ properties.setProperty(DatabaseProperties.PROP_URL, ((BasicDataSource) dbTester.database().getDataSource()).getUrl());
properties.setProperty(DatabaseProperties.PROP_USER, "sonar");
properties.setProperty(DatabaseProperties.PROP_PASSWORD, "sonar");
+
+ // required persisted properties
insertProperty(CoreProperties.SERVER_ID, "a_startup_id");
insertProperty(CoreProperties.SERVER_STARTTIME, DateUtils.formatDateTime(new Date()));
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
/**
* Constants shared by search, web server and app processes.
* They are almost all the properties defined in conf/sonar.properties.
*/
public class ProcessProperties {
- public static final String CLUSTER_ACTIVATE = "sonar.cluster.activate";
- public static final String CLUSTER_MASTER = "sonar.cluster.master";
- public static final String CLUSTER_MASTER_HOST = "sonar.cluster.masterHost";
- public static final String CLUSTER_NAME = "sonar.cluster.name";
- public static final String CLUSTER_NODE_NAME = "sonar.node.name";
+ public static final String CLUSTER_ENABLED = "sonar.cluster.enabled";
public static final String CLUSTER_CE_DISABLED = "sonar.cluster.ce.disabled";
+ public static final String CLUSTER_SEARCH_DISABLED = "sonar.cluster.search.disabled";
+ public static final String CLUSTER_SEARCH_HOSTS = "sonar.cluster.search.hosts";
public static final String CLUSTER_WEB_DISABLED = "sonar.cluster.web.disabled";
public static final String JDBC_URL = "sonar.jdbc.url";
public static final String PATH_TEMP = "sonar.path.temp";
public static final String PATH_WEB = "sonar.path.web";
+ public static final String SEARCH_CLUSTER_NAME = "sonar.search.clusterName";
public static final String SEARCH_HOST = "sonar.search.host";
public static final String SEARCH_PORT = "sonar.search.port";
public static final String SEARCH_HTTP_PORT = "sonar.search.httpPort";
public static final String SEARCH_JAVA_ADDITIONAL_OPTS = "sonar.search.javaAdditionalOpts";
public static final String WEB_JAVA_OPTS = "sonar.web.javaOpts";
- public static final String WEB_JAVA_ADDITIONAL_OPTS = "sonar.web.javaAdditionalOpts";
+ public static final String WEB_JAVA_ADDITIONAL_OPTS = "sonar.web.javaAdditionalOpts";
public static final String CE_JAVA_OPTS = "sonar.ce.javaOpts";
public static final String CE_JAVA_ADDITIONAL_OPTS = "sonar.ce.javaAdditionalOpts";
public static final String ENABLE_STOP_COMMAND = "sonar.enableStopCommand";
public static final String WEB_ENFORCED_JVM_ARGS = "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djruby.management.enabled=false " +
- // jruby is slow with java 8: https://jira.sonarsource.com/browse/SONAR-6115
+ // jruby is slow with java 8: https://jira.sonarsource.com/browse/SONAR-6115
"-Djruby.compile.invokedynamic=false";
public static final String CE_ENFORCED_JVM_ARGS = "-Djava.awt.headless=true -Dfile.encoding=UTF-8";
public static void completeDefaults(Props props) {
// init string properties
- for (Map.Entry<String, String> entry : defaults().entrySet()) {
- props.setDefault(entry.getKey(), entry.getValue());
+ for (Map.Entry<Object, Object> entry : defaults().entrySet()) {
+ props.setDefault(entry.getKey().toString(), entry.getValue().toString());
}
// init ports
}
}
- public static Map<String, String> defaults() {
- Map<String, String> defaults = new HashMap<>();
- defaults.put(ProcessProperties.CLUSTER_NAME, "sonarqube");
- defaults.put(ProcessProperties.CLUSTER_NODE_NAME, "sonar-" + System.currentTimeMillis());
- defaults.put(ProcessProperties.CLUSTER_CE_DISABLED, "false");
- defaults.put(ProcessProperties.CLUSTER_WEB_DISABLED, "false");
-
+ public static Properties defaults() {
+ Properties defaults = new Properties();
+ defaults.put(ProcessProperties.SEARCH_CLUSTER_NAME, "sonarqube");
defaults.put(ProcessProperties.SEARCH_HOST, "127.0.0.1");
defaults.put(ProcessProperties.SEARCH_JAVA_OPTS, "-Xmx1G -Xms256m -Xss256k -Djna.nosys=true " +
"-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly " +
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
-import java.util.LinkedHashSet;
import java.util.Set;
import java.util.TreeSet;
+import java.util.UUID;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.sonar.process.MessageException;
import org.sonar.process.ProcessProperties;
import org.sonar.process.Props;
public class EsSettings implements EsSettingsMBean {
private static final Logger LOGGER = LoggerFactory.getLogger(EsSettings.class);
-
public static final String PROP_MARVEL_HOSTS = "sonar.search.marvelHosts";
+ public static final String CLUSTER_SEARCH_NODE_NAME = "sonar.cluster.search.nodeName";
+ public static final String STANDALONE_NODE_NAME = "sonarqube";
private final Props props;
- private final Set<String> masterHosts = new LinkedHashSet<>();
+
+ private final boolean clusterEnabled;
+ private final String clusterName;
+ private final String nodeName;
EsSettings(Props props) {
this.props = props;
- masterHosts.addAll(Arrays.asList(StringUtils.split(props.value(ProcessProperties.CLUSTER_MASTER_HOST, ""), ",")));
- }
+ // name of ES cluster must always be set, even if clustering of SQ is disabled
+ this.clusterName = props.nonNullValue(ProcessProperties.SEARCH_CLUSTER_NAME);
- boolean inCluster() {
- return props.valueAsBoolean(ProcessProperties.CLUSTER_ACTIVATE, false);
- }
-
- boolean isMaster() {
- return props.valueAsBoolean(ProcessProperties.CLUSTER_MASTER, false);
+ this.clusterEnabled = props.valueAsBoolean(ProcessProperties.CLUSTER_ENABLED);
+ if (this.clusterEnabled) {
+ this.nodeName = props.value(CLUSTER_SEARCH_NODE_NAME, "sonarqube-" + UUID.randomUUID().toString());
+ } else {
+ this.nodeName = STANDALONE_NODE_NAME;
+ }
}
@Override
@Override
public String getClusterName() {
- return props.value(ProcessProperties.CLUSTER_NAME);
+ return clusterName;
}
@Override
public String getNodeName() {
- return props.value(ProcessProperties.CLUSTER_NODE_NAME);
+ return nodeName;
}
Settings build() {
// standard configuration
builder.put("http.enabled", false);
} else {
- LOGGER.warn(String.format(
- "Elasticsearch HTTP connector is enabled on port %d. MUST NOT BE USED FOR PRODUCTION", httpPort));
+ LOGGER.warn("Elasticsearch HTTP connector is enabled on port {}. MUST NOT BE USED FOR PRODUCTION", httpPort);
// see https://github.com/lmenezes/elasticsearch-kopf/issues/195
builder.put("http.cors.enabled", true);
builder.put("http.enabled", true);
private void configureCluster(Settings.Builder builder) {
int replicationFactor = 0;
- if (inCluster()) {
+ if (clusterEnabled) {
replicationFactor = 1;
- if (isMaster()) {
- LOGGER.info("Elasticsearch cluster enabled. Master node.");
- builder.put("node.master", true);
- } else if (!masterHosts.isEmpty()) {
- LOGGER.info("Elasticsearch cluster enabled. Node connecting to master: {}", masterHosts);
- builder.put("discovery.zen.ping.unicast.hosts", StringUtils.join(masterHosts, ","));
- builder.put("node.master", false);
- builder.put("discovery.zen.minimum_master_nodes", 1);
- } else {
- throw new MessageException(String.format("Not an Elasticsearch master nor slave. Please check properties %s and %s",
- ProcessProperties.CLUSTER_MASTER, ProcessProperties.CLUSTER_MASTER_HOST));
- }
+ String hosts = props.value(ProcessProperties.CLUSTER_SEARCH_HOSTS, "");
+ LOGGER.info("Elasticsearch cluster enabled. Connect to hosts [{}]", hosts);
+ builder.put("discovery.zen.ping.unicast.hosts", hosts);
}
+ builder.put("discovery.zen.minimum_master_nodes", 1);
builder.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicationFactor);
builder.put("cluster.name", getClusterName());
builder.put("cluster.routing.allocation.awareness.attributes", "rack_id");
String nodeName = getNodeName();
builder.put("node.rack_id", nodeName);
builder.put("node.name", nodeName);
+ builder.put("node.data", true);
+ builder.put("node.master", true);
}
private void configureMarvel(Settings.Builder builder) {
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import org.sonar.process.MessageException;
import org.sonar.process.ProcessProperties;
import org.sonar.process.Props;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.fail;
public class EsSettingsTest {
props.set(ProcessProperties.SEARCH_PORT, "1234");
props.set(ProcessProperties.SEARCH_HOST, "127.0.0.1");
props.set(ProcessProperties.PATH_HOME, homeDir.getAbsolutePath());
- props.set(ProcessProperties.CLUSTER_NAME, "tests");
- props.set(ProcessProperties.CLUSTER_NODE_NAME, "test");
+ props.set(ProcessProperties.SEARCH_CLUSTER_NAME, "sonarqube");
EsSettings esSettings = new EsSettings(props);
- assertThat(esSettings.inCluster()).isFalse();
Settings generated = esSettings.build();
assertThat(generated.get("transport.tcp.port")).isEqualTo("1234");
assertThat(generated.get("transport.host")).isEqualTo("127.0.0.1");
- assertThat(generated.get("cluster.name")).isEqualTo("tests");
- assertThat(generated.get("node.name")).isEqualTo("test");
+
+ // no cluster, but cluster and node names are set though
+ assertThat(generated.get("cluster.name")).isEqualTo("sonarqube");
+ assertThat(generated.get("node.name")).isEqualTo("sonarqube");
assertThat(generated.get("path.data")).isNotNull();
assertThat(generated.get("path.logs")).isNotNull();
// http is disabled for security reasons
assertThat(generated.get("http.enabled")).isEqualTo("false");
- // no cluster, but node name is set though
assertThat(generated.get("index.number_of_replicas")).isEqualTo("0");
assertThat(generated.get("discovery.zen.ping.unicast.hosts")).isNull();
}
-
- @Test
- public void test_default_hosts() throws Exception {
- Props props = minProps();
-
- EsSettings esSettings = new EsSettings(props);
- assertThat(esSettings.inCluster()).isFalse();
-
- Settings generated = esSettings.build();
- assertThat(generated.get("transport.tcp.port")).isEqualTo("9001");
- assertThat(generated.get("transport.host")).isEqualTo("127.0.0.1");
- assertThat(generated.get("cluster.name")).isEqualTo("sonarqube");
- assertThat(generated.get("node.name")).startsWith("sonar-");
- }
@Test
public void override_dirs() throws Exception {
}
@Test
- public void test_cluster_master() throws Exception {
+ public void cluster_is_enabled() throws Exception {
Props props = minProps();
- props.set(ProcessProperties.CLUSTER_ACTIVATE, "true");
- props.set(ProcessProperties.CLUSTER_MASTER, "true");
+ props.set(ProcessProperties.CLUSTER_ENABLED, "true");
+ props.set(ProcessProperties.CLUSTER_SEARCH_HOSTS, "1.2.3.4:9000,1.2.3.5:8080");
Settings settings = new EsSettings(props).build();
assertThat(settings.get("index.number_of_replicas")).isEqualTo("1");
- assertThat(settings.get("discovery.zen.ping.unicast.hosts")).isNull();
- assertThat(settings.get("node.master")).isEqualTo("true");
- }
-
- @Test
- public void test_cluster_slave() throws Exception {
- Props props = minProps();
- props.set(ProcessProperties.CLUSTER_ACTIVATE, "true");
- props.set(ProcessProperties.CLUSTER_MASTER_HOST, "127.0.0.2,127.0.0.3");
- Settings settings = new EsSettings(props).build();
-
- assertThat(settings.get("discovery.zen.ping.unicast.hosts")).isEqualTo("127.0.0.2,127.0.0.3");
- assertThat(settings.get("node.master")).isEqualTo("false");
- }
-
- @Test
- public void bad_cluster_configuration() throws Exception {
- Props props = minProps();
- props.set(ProcessProperties.CLUSTER_ACTIVATE, "true");
- try {
- new EsSettings(props).build();
- fail();
- } catch (MessageException ignored) {
- // expected
- }
+ assertThat(settings.get("discovery.zen.ping.unicast.hosts")).isEqualTo("1.2.3.4:9000,1.2.3.5:8080");
}
@Test
public class SearchServerTest {
- static final String CLUSTER_NAME = "unitTest";
-
- int port = NetworkUtils.freePort();
- SearchServer searchServer;
- Client client;
+ private static final String A_CLUSTER_NAME = "a_cluster";
+ private static final String A_NODE_NAME = "a_node";
@Rule
public TestRule timeout = new DisableOnDebug(Timeout.seconds(60));
@Rule
public TemporaryFolder temp = new TemporaryFolder();
+ private int port = NetworkUtils.freePort();
+ private Client client;
+ private SearchServer underTest;
+
@After
public void tearDown() {
- if (searchServer != null) {
- searchServer.stop();
- searchServer.awaitStop();
+ if (underTest != null) {
+ underTest.stop();
+ underTest.awaitStop();
}
if (client != null) {
client.close();
InetAddress host = InetAddress.getLocalHost();
props.set(ProcessProperties.SEARCH_HOST, host.getHostAddress());
props.set(ProcessProperties.SEARCH_PORT, String.valueOf(port));
- props.set(ProcessProperties.CLUSTER_NAME, CLUSTER_NAME);
- props.set(ProcessProperties.CLUSTER_NODE_NAME, "test");
+ props.set(ProcessProperties.SEARCH_CLUSTER_NAME, A_CLUSTER_NAME);
+ props.set(EsSettings.CLUSTER_SEARCH_NODE_NAME, A_NODE_NAME);
props.set(ProcessProperties.PATH_HOME, temp.newFolder().getAbsolutePath());
props.set(ProcessEntryPoint.PROPERTY_SHARED_PATH, temp.newFolder().getAbsolutePath());
- searchServer = new SearchServer(props);
- searchServer.start();
- assertThat(searchServer.isUp()).isTrue();
+ underTest = new SearchServer(props);
+ underTest.start();
+ assertThat(underTest.isUp()).isTrue();
- Settings settings = Settings.builder().put("cluster.name", CLUSTER_NAME).build();
+ Settings settings = Settings.builder().put("cluster.name", A_CLUSTER_NAME).build();
client = TransportClient.builder().settings(settings).build()
.addTransportAddress(new InetSocketTransportAddress(host, port));
assertThat(client.admin().cluster().prepareClusterStats().get().getStatus()).isEqualTo(ClusterHealthStatus.GREEN);
- searchServer.stop();
- searchServer.awaitStop();
- searchServer = null;
+ underTest.stop();
+ underTest.awaitStop();
+ underTest = null;
try {
client.admin().cluster().prepareClusterStats().get();
fail();
*/
package org.sonar.server.es;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.picocontainer.Startable;
-import org.sonar.api.config.Settings;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
-import org.sonar.process.ProcessProperties;
import org.sonar.server.es.request.ProxyBulkRequestBuilder;
import org.sonar.server.es.request.ProxyClearCacheRequestBuilder;
import org.sonar.server.es.request.ProxyClusterHealthRequestBuilder;
import org.sonar.server.es.request.ProxySearchRequestBuilder;
import org.sonar.server.es.request.ProxySearchScrollRequestBuilder;
-import static org.apache.commons.lang.StringUtils.defaultIfEmpty;
+import static java.util.Objects.requireNonNull;
/**
* Facade to connect to Elasticsearch node. Handles correctly errors (logging + exceptions
public class EsClient implements Startable {
public static final Logger LOGGER = Loggers.get("es");
- private final Settings settings;
- private Client nativeClient = null;
- public EsClient(Settings settings) {
- this.settings = settings;
- }
+ private final Client nativeClient;
- EsClient(Settings settings, Client nativeClient) {
- this.settings = settings;
- this.nativeClient = nativeClient;
+ public EsClient(Client nativeClient) {
+ this.nativeClient = requireNonNull(nativeClient);
}
public RefreshRequestBuilder prepareRefresh(String... indices) {
@Override
public void start() {
- if (nativeClient == null) {
- ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory());
- org.elasticsearch.common.settings.Settings esSettings = org.elasticsearch.common.settings.Settings.builder()
- .put("node.name", defaultIfEmpty(settings.getString(ProcessProperties.CLUSTER_NODE_NAME), "sq_local_client"))
- .put("node.rack_id", defaultIfEmpty(settings.getString(ProcessProperties.CLUSTER_NODE_NAME), "unknown"))
- .put("cluster.name", StringUtils.defaultIfBlank(settings.getString(ProcessProperties.CLUSTER_NAME), "sonarqube"))
- .build();
- nativeClient = TransportClient.builder().settings(esSettings).build();
- String host = settings.getString(ProcessProperties.SEARCH_HOST);
- try {
- ((TransportClient) nativeClient).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), settings.getInt(ProcessProperties.SEARCH_PORT)));
- } catch (UnknownHostException e) {
- throw new IllegalStateException("Can not resolve host [" + host + "]", e);
- }
- }
+ // nothing to do
}
@Override
public void stop() {
- if (nativeClient != null) {
- nativeClient.close();
- }
+ nativeClient.close();
}
public Client nativeClient() {
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import javax.annotation.concurrent.Immutable;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.picocontainer.injectors.ProviderAdapter;
+import org.sonar.api.ce.ComputeEngineSide;
+import org.sonar.api.config.Settings;
+import org.sonar.api.server.ServerSide;
+import org.sonar.process.ProcessProperties;
+
+@ComputeEngineSide
+@ServerSide
+public class EsClientProvider extends ProviderAdapter {
+
+ private EsClient cache;
+
+ public EsClient provide(Settings settings) {
+ if (cache == null) {
+ TransportClient nativeClient;
+ org.elasticsearch.common.settings.Settings.Builder esSettings = org.elasticsearch.common.settings.Settings.builder();
+
+ // mandatory property defined by bootstrap process
+ esSettings.put("cluster.name", settings.getString(ProcessProperties.SEARCH_CLUSTER_NAME));
+
+ boolean clusterEnabled = settings.getBoolean(ProcessProperties.CLUSTER_ENABLED);
+ if (clusterEnabled) {
+ esSettings.put("client.transport.sniff", true);
+ nativeClient = TransportClient.builder().settings(esSettings).build();
+ Arrays.stream(settings.getStringArray(ProcessProperties.CLUSTER_SEARCH_HOSTS))
+ .map(Host::parse)
+ .forEach(h -> h.addTo(nativeClient));
+ } else {
+ nativeClient = TransportClient.builder().settings(esSettings).build();
+ Host host = new Host(settings.getString(ProcessProperties.SEARCH_HOST), settings.getInt(ProcessProperties.SEARCH_PORT));
+ host.addTo(nativeClient);
+ }
+ cache = new EsClient(nativeClient);
+ }
+ return cache;
+ }
+
+ @Immutable
+ private static class Host {
+ private final String ip;
+ private final int port;
+
+ Host(String ip, int port) {
+ this.ip = ip.trim();
+ this.port = port;
+ }
+
+ static Host parse(String s) {
+ String[] split = s.split(":");
+ if (split.length != 2) {
+ throw new IllegalArgumentException("Badly formatted Elasticsearch host: " + s);
+ }
+ return new Host(split[0], Integer.parseInt(split[1]));
+ }
+
+ void addTo(TransportClient client) {
+ try {
+ client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ip), port));
+ } catch (UnknownHostException e) {
+ throw new IllegalStateException("Can not resolve host [" + ip + "]", e);
+ }
+ }
+ }
+}
}
public void configureShards(org.sonar.api.config.Settings settings) {
- boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE);
+ boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ENABLED);
int shards = settings.getInt(format("sonar.search.%s.shards", indexName));
if (shards == 0) {
shards = DEFAULT_NUMBER_OF_SHARDS;
package org.sonar.server.search;
import org.sonar.core.platform.Module;
-import org.sonar.server.es.EsClient;
+import org.sonar.server.es.EsClientProvider;
public class EsSearchModule extends Module {
@Override
protected void configureModule() {
- add(EsClient.class);
+ add(new EsClientProvider());
}
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import java.net.InetAddress;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.sonar.api.config.Settings;
+import org.sonar.process.ProcessProperties;
+
+import static java.lang.String.format;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class EsClientProviderTest {
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ private Settings settings = new Settings();
+ private EsClientProvider underTest = new EsClientProvider();
+ private String localhost;
+
+ @Before
+ public void setUp() throws Exception {
+ // mandatory property
+ settings.setProperty(ProcessProperties.SEARCH_CLUSTER_NAME, "the_cluster_name");
+
+ localhost = InetAddress.getLocalHost().getHostAddress();
+ }
+
+ @Test
+ public void connection_to_local_es_when_cluster_mode_is_disabled() throws Exception {
+ settings.setProperty(ProcessProperties.CLUSTER_ENABLED, false);
+ settings.setProperty(ProcessProperties.SEARCH_HOST, localhost);
+ settings.setProperty(ProcessProperties.SEARCH_PORT, 8080);
+
+ EsClient client = underTest.provide(settings);
+ TransportClient transportClient = (TransportClient) client.nativeClient();
+ assertThat(transportClient.transportAddresses()).hasSize(1);
+ TransportAddress address = transportClient.transportAddresses().get(0);
+ assertThat(address.getAddress()).isEqualTo(localhost);
+ assertThat(address.getPort()).isEqualTo(8080);
+
+ // keep in cache
+ assertThat(underTest.provide(settings)).isSameAs(client);
+ }
+
+ @Test
+ public void connection_to_remote_es_nodes_when_cluster_mode_is_enabled() throws Exception {
+ settings.setProperty(ProcessProperties.CLUSTER_ENABLED, true);
+ settings.setProperty(ProcessProperties.CLUSTER_SEARCH_HOSTS, format("%s:8080,%s:8081", localhost, localhost));
+
+ EsClient client = underTest.provide(settings);
+ TransportClient transportClient = (TransportClient) client.nativeClient();
+ assertThat(transportClient.transportAddresses()).hasSize(2);
+ TransportAddress address = transportClient.transportAddresses().get(0);
+ assertThat(address.getAddress()).isEqualTo(localhost);
+ assertThat(address.getPort()).isEqualTo(8080);
+ address = transportClient.transportAddresses().get(1);
+ assertThat(address.getAddress()).isEqualTo(localhost);
+ assertThat(address.getPort()).isEqualTo(8081);
+
+ // keep in cache
+ assertThat(underTest.provide(settings)).isSameAs(client);
+ }
+
+ @Test
+ public void fail_if_cluster_host_is_badly_formatted() throws Exception {
+ settings.setProperty(ProcessProperties.CLUSTER_ENABLED, true);
+ settings.setProperty(ProcessProperties.CLUSTER_SEARCH_HOSTS, "missing_colon");
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Badly formatted Elasticsearch host: missing_colon");
+ underTest.provide(settings);
+ }
+}
private static EsServerHolder HOLDER = null;
private final String clusterName;
- private final String nodeName;
private final int port;
private final String hostName;
private final File homeDir;
private final SearchServer server;
- private EsServerHolder(SearchServer server, String clusterName, String nodeName, int port, String hostName, File homeDir) {
+ private EsServerHolder(SearchServer server, String clusterName, int port, String hostName, File homeDir) {
this.server = server;
this.clusterName = clusterName;
- this.nodeName = nodeName;
this.port = port;
this.hostName = hostName;
this.homeDir = homeDir;
return clusterName;
}
- public String getNodeName() {
- return nodeName;
- }
-
public int getPort() {
return port;
}
private void reset() {
TransportClient client = TransportClient.builder().settings(Settings.builder()
- .put("node.name", nodeName)
.put("network.bind_host", "localhost")
.put("cluster.name", clusterName)
.build()).build();
homeDir.mkdir();
String clusterName = "testCluster";
- String nodeName = "test";
String hostName = "127.0.0.1";
int port = NetworkUtils.freePort();
Properties properties = new Properties();
- properties.setProperty(ProcessProperties.CLUSTER_NAME, clusterName);
- properties.setProperty(ProcessProperties.CLUSTER_NODE_NAME, nodeName);
+ properties.setProperty(ProcessProperties.SEARCH_CLUSTER_NAME, clusterName);
properties.setProperty(ProcessProperties.SEARCH_PORT, String.valueOf(port));
properties.setProperty(ProcessProperties.SEARCH_HOST, hostName);
properties.setProperty(ProcessProperties.PATH_HOME, homeDir.getAbsolutePath());
properties.setProperty(ProcessEntryPoint.PROPERTY_SHARED_PATH, homeDir.getAbsolutePath());
SearchServer server = new SearchServer(new Props(properties));
server.start();
- HOLDER = new EsServerHolder(server, clusterName, nodeName, port, hostName, homeDir);
+ HOLDER = new EsServerHolder(server, clusterName, port, hostName, homeDir);
}
HOLDER.reset();
return HOLDER;
public class EsTester extends ExternalResource {
private final List<IndexDefinition> indexDefinitions;
- private EsClient client = new EsClient(new Settings(), NodeHolder.INSTANCE.node.client());
+ private EsClient client = new EsClient(NodeHolder.INSTANCE.node.client());
private ComponentContainer container;
public EsTester(IndexDefinition... defs) {
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
+import org.sonar.process.ProcessProperties;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
public void five_shards_and_one_replica_by_default_on_cluster() {
NewIndex index = new NewIndex("issues");
org.sonar.api.config.Settings settings = new org.sonar.api.config.Settings();
- settings.setProperty("sonar.cluster.activate", "true");
+ settings.setProperty(ProcessProperties.CLUSTER_ENABLED, "true");
index.configureShards(settings);
assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo(String.valueOf(NewIndex.DEFAULT_NUMBER_OF_SHARDS));
assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1");
@Test
public void enable_replica_if_clustering_is_enabled() {
- settings.setProperty(ProcessProperties.CLUSTER_ACTIVATE, true);
+ settings.setProperty(ProcessProperties.CLUSTER_ENABLED, true);
IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
underTest.define(context);
public void support_long_html_description() throws Exception {
String longText = StringUtils.repeat("hello ", 10_000);
// the following method fails if PUT fails
- tester.putDocuments(INDEX, RuleIndexDefinition.TYPE_RULE, new RuleDoc(ImmutableMap.<String, Object>of(
+ tester.putDocuments(INDEX, RuleIndexDefinition.TYPE_RULE, new RuleDoc(ImmutableMap.of(
FIELD_RULE_HTML_DESCRIPTION, longText,
FIELD_RULE_REPOSITORY, "squid",
FIELD_RULE_KEY, "squid:S001")));
Properties properties = new Properties();
properties.putAll(initialProps);
esServerHolder = EsServerHolder.get();
- properties.setProperty(ProcessProperties.CLUSTER_NAME, esServerHolder.getClusterName());
- properties.setProperty(ProcessProperties.CLUSTER_NODE_NAME, esServerHolder.getNodeName());
+ properties.setProperty(ProcessProperties.SEARCH_CLUSTER_NAME, esServerHolder.getClusterName());
properties.setProperty(ProcessProperties.SEARCH_PORT, String.valueOf(esServerHolder.getPort()));
properties.setProperty(ProcessProperties.SEARCH_HOST, String.valueOf(esServerHolder.getHostName()));
properties.setProperty(ProcessProperties.PATH_HOME, homeDir.getAbsolutePath());
import java.util.List;
import java.util.Properties;
import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang.StringUtils;
import org.sonar.process.ProcessId;
import org.sonar.process.ProcessProperties;
import org.sonar.process.Props;
private static List<JavaCommand> createCommands(Props props) {
File homeDir = props.nonNullValueAsFile(ProcessProperties.PATH_HOME);
List<JavaCommand> commands = new ArrayList<>(3);
- commands.add(createESCommand(props, homeDir));
+ if (isProcessEnabled(props, ProcessProperties.CLUSTER_SEARCH_DISABLED)) {
+ commands.add(createESCommand(props, homeDir));
+ }
- // do not yet start WebServer nor CE on elasticsearch slaves
- if (StringUtils.isBlank(props.value(ProcessProperties.CLUSTER_MASTER_HOST))) {
- if (!props.valueAsBoolean(ProcessProperties.CLUSTER_WEB_DISABLED)) {
- commands.add(createWebServerCommand(props, homeDir));
- }
+ if (isProcessEnabled(props, ProcessProperties.CLUSTER_WEB_DISABLED)) {
+ commands.add(createWebServerCommand(props, homeDir));
+ }
- if (!props.valueAsBoolean(ProcessProperties.CLUSTER_CE_DISABLED)) {
- commands.add(createCeServerCommand(props, homeDir));
- }
+ if (isProcessEnabled(props, ProcessProperties.CLUSTER_CE_DISABLED)) {
+ commands.add(createCeServerCommand(props, homeDir));
}
return commands;
}
+ private static boolean isProcessEnabled(Props props, String disabledPropertyKey) {
+ return !props.valueAsBoolean(ProcessProperties.CLUSTER_ENABLED) ||
+ !props.valueAsBoolean(disabledPropertyKey);
+ }
+
private static JavaCommand createESCommand(Props props, File homeDir) {
JavaCommand elasticsearch = new JavaCommand(ProcessId.ELASTICSEARCH);
elasticsearch
}
@Test
- public void start_all_processes_by_default() throws Exception {
+ public void start_all_processes_if_cluster_mode_is_disabled() throws Exception {
Monitor monitor = mock(Monitor.class);
App app = new App(monitor);
Props props = initDefaultProps();
verify(monitor).start(argument.capture());
assertThat(argument.getValue()).extracting("processId").containsExactly(ProcessId.ELASTICSEARCH, ProcessId.WEB_SERVER, ProcessId.COMPUTE_ENGINE);
+
+ app.stopAsync();
+ }
+
+ @Test
+ public void start_only_web_server_node_in_cluster() throws Exception {
+ Monitor monitor = mock(Monitor.class);
+ App app = new App(monitor);
+ Props props = initDefaultProps();
+ props.set(ProcessProperties.CLUSTER_ENABLED, "true");
+ props.set(ProcessProperties.CLUSTER_CE_DISABLED, "true");
+ props.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true");
+ app.start(props);
+
+ ArgumentCaptor<List<JavaCommand>> argument = newJavaCommandArgumentCaptor();
+ verify(monitor).start(argument.capture());
+
+ assertThat(argument.getValue()).extracting("processId").containsOnly(ProcessId.WEB_SERVER);
+ }
+
+ @Test
+ public void start_only_compute_engine_node_in_cluster() throws Exception {
+ Monitor monitor = mock(Monitor.class);
+ App app = new App(monitor);
+ Props props = initDefaultProps();
+ props.set(ProcessProperties.CLUSTER_ENABLED, "true");
+ props.set(ProcessProperties.CLUSTER_WEB_DISABLED, "true");
+ props.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true");
+ app.start(props);
+
+ ArgumentCaptor<List<JavaCommand>> argument = newJavaCommandArgumentCaptor();
+ verify(monitor).start(argument.capture());
+
+ assertThat(argument.getValue()).extracting("processId").containsOnly(ProcessId.COMPUTE_ENGINE);
}
@Test
- public void do_not_start_WebServer_nor_CE_if_elasticsearch_slave() throws Exception {
+ public void start_only_elasticsearch_node_in_cluster() throws Exception {
Monitor monitor = mock(Monitor.class);
App app = new App(monitor);
Props props = initDefaultProps();
- props.set("sonar.cluster.masterHost", "1.2.3.4");
+ props.set(ProcessProperties.CLUSTER_ENABLED, "true");
+ props.set(ProcessProperties.CLUSTER_WEB_DISABLED, "true");
+ props.set(ProcessProperties.CLUSTER_CE_DISABLED, "true");
app.start(props);
ArgumentCaptor<List<JavaCommand>> argument = newJavaCommandArgumentCaptor();
projects / sonarqube.git / commitdiff