package org.elasticsearch.client:transport and most of its dependencies into sonar-application.jar; do not open Elasticsearch's http port by defaulttags/6.6-RC1
@@ -688,11 +688,6 @@ | |||
</exclusion> | |||
</exclusions> | |||
</dependency> | |||
<dependency> | |||
<groupId>net.java.dev.jna</groupId> | |||
<artifactId>jna</artifactId> | |||
<version>4.1.0</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>com.hazelcast</groupId> | |||
<artifactId>hazelcast</artifactId> | |||
@@ -704,8 +699,8 @@ | |||
<version>${hazelcast.version}</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.elasticsearch</groupId> | |||
<artifactId>elasticsearch</artifactId> | |||
<groupId>org.elasticsearch.client</groupId> | |||
<artifactId>transport</artifactId> | |||
<version>${elasticsearch.version}</version> | |||
</dependency> | |||
<dependency> |
@@ -47,6 +47,15 @@ | |||
<groupId>com.google.guava</groupId> | |||
<artifactId>guava</artifactId> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.elasticsearch.client</groupId> | |||
<artifactId>transport</artifactId> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.apache.logging.log4j</groupId> | |||
<artifactId>log4j-api</artifactId> | |||
<version>2.6.2</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>com.google.code.findbugs</groupId> |
@@ -107,7 +107,7 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi | |||
private void tryToStartEs() { | |||
SQProcess process = processesById.get(ProcessId.ELASTICSEARCH); | |||
if (process != null) { | |||
tryToStartEsProcess(process, commandFactory::createEsCommand); | |||
tryToStartEsProcess(process, () -> commandFactory.createEsCommand(settings)); | |||
} | |||
} | |||
@@ -19,9 +19,11 @@ | |||
*/ | |||
package org.sonar.application.process; | |||
import org.sonar.application.config.AppSettings; | |||
public interface CommandFactory { | |||
EsCommand createEsCommand(); | |||
EsCommand createEsCommand(AppSettings settings); | |||
JavaCommand createWebCommand(boolean leader); | |||
@@ -52,22 +52,24 @@ public class CommandFactoryImpl implements CommandFactory { | |||
} | |||
@Override | |||
public EsCommand createEsCommand() { | |||
File homeDir = settings.getProps().nonNullValueAsFile(ProcessProperties.PATH_HOME); | |||
public EsCommand createEsCommand(AppSettings settings) { | |||
File homeDir = this.settings.getProps().nonNullValueAsFile(ProcessProperties.PATH_HOME); | |||
File executable = new File(homeDir, getExecutable()); | |||
if (!executable.exists()) { | |||
throw new IllegalStateException("Cannot find elasticsearch binary"); | |||
} | |||
Map<String, String> settingsMap = new EsSettings(settings.getProps()).build(); | |||
Map<String, String> settingsMap = new EsSettings(this.settings.getProps()).build(); | |||
EsCommand res = new EsCommand(ProcessId.ELASTICSEARCH) | |||
.setWorkDir(executable.getParentFile().getParentFile()) | |||
.setExecutable(executable) | |||
.setArguments(settings.getProps().rawProperties()) | |||
.setArguments(this.settings.getProps().rawProperties()) | |||
.setClusterName(settingsMap.get("cluster.name")) | |||
.setHost(settingsMap.get("network.host")) | |||
// TODO add argument to specify log4j configuration file | |||
// TODO add argument to specify yaml configuration file | |||
.setUrl("http://" + settingsMap.get("http.host") + ":" + settingsMap.get("http.port")); | |||
.setPort(Integer.valueOf(settingsMap.get("transport.tcp.port"))); | |||
settingsMap.entrySet().stream() | |||
.filter(entry -> !"path.home".equals(entry.getKey())) |
@@ -26,7 +26,9 @@ import org.sonar.process.ProcessId; | |||
public class EsCommand extends AbstractCommand<EsCommand> { | |||
private File executable; | |||
private String url; | |||
private String clusterName; | |||
private String host; | |||
private int port; | |||
private List<String> esOptions = new ArrayList<>(); | |||
public EsCommand(ProcessId id) { | |||
@@ -42,12 +44,30 @@ public class EsCommand extends AbstractCommand<EsCommand> { | |||
return this; | |||
} | |||
public String getUrl() { | |||
return url; | |||
public String getClusterName() { | |||
return clusterName; | |||
} | |||
public EsCommand setUrl(String url) { | |||
this.url = url; | |||
public EsCommand setClusterName(String clusterName) { | |||
this.clusterName = clusterName; | |||
return this; | |||
} | |||
public String getHost() { | |||
return host; | |||
} | |||
public EsCommand setHost(String host) { | |||
this.host = host; | |||
return this; | |||
} | |||
public int getPort() { | |||
return port; | |||
} | |||
public EsCommand setPort(int port) { | |||
this.port = port; | |||
return this; | |||
} | |||
@@ -19,16 +19,38 @@ | |||
*/ | |||
package org.sonar.application.process; | |||
import java.io.IOException; | |||
import java.net.ConnectException; | |||
import com.google.common.net.HostAndPort; | |||
import io.netty.util.ThreadDeathWatcher; | |||
import io.netty.util.concurrent.GlobalEventExecutor; | |||
import java.net.InetAddress; | |||
import java.net.MalformedURLException; | |||
import java.net.URL; | |||
import java.net.URLConnection; | |||
import java.net.UnknownHostException; | |||
import java.util.concurrent.TimeUnit; | |||
import java.util.concurrent.atomic.AtomicBoolean; | |||
import org.apache.commons.io.IOUtils; | |||
import java.util.concurrent.atomic.AtomicReference; | |||
import java.util.stream.Collectors; | |||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; | |||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; | |||
import org.elasticsearch.client.transport.NoNodeAvailableException; | |||
import org.elasticsearch.client.transport.TransportClient; | |||
import org.elasticsearch.cluster.health.ClusterHealthStatus; | |||
import org.elasticsearch.common.network.NetworkModule; | |||
import org.elasticsearch.common.settings.Settings; | |||
import org.elasticsearch.common.transport.InetSocketTransportAddress; | |||
import org.elasticsearch.common.transport.TransportAddress; | |||
import org.elasticsearch.transport.Netty4Plugin; | |||
import org.slf4j.Logger; | |||
import org.slf4j.LoggerFactory; | |||
import static java.util.Collections.singletonList; | |||
import static java.util.Collections.unmodifiableList; | |||
import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; | |||
import static org.sonar.application.process.EsProcessMonitor.Status.CONNECTION_REFUSED; | |||
import static org.sonar.application.process.EsProcessMonitor.Status.GREEN; | |||
import static org.sonar.application.process.EsProcessMonitor.Status.KO; | |||
import static org.sonar.application.process.EsProcessMonitor.Status.RED; | |||
import static org.sonar.application.process.EsProcessMonitor.Status.YELLOW; | |||
public class EsProcessMonitor extends AbstractProcessMonitor { | |||
private static final Logger LOG = LoggerFactory.getLogger(EsProcessMonitor.class); | |||
private static final int WAIT_FOR_UP_DELAY_IN_MILLIS = 100; | |||
@@ -36,11 +58,12 @@ public class EsProcessMonitor extends AbstractProcessMonitor { | |||
private final AtomicBoolean nodeUp = new AtomicBoolean(false); | |||
private final AtomicBoolean nodeOperational = new AtomicBoolean(false); | |||
private final URL healthCheckURL; | |||
private final EsCommand esCommand; | |||
private AtomicReference<TransportClient> transportClient = new AtomicReference<>(null); | |||
public EsProcessMonitor(Process process, String url) throws MalformedURLException { | |||
public EsProcessMonitor(Process process, EsCommand esCommand) throws MalformedURLException { | |||
super(process); | |||
this.healthCheckURL = new URL(url + "/_cluster/health?wait_for_status=yellow&timeout=30s"); | |||
this.esCommand = esCommand; | |||
} | |||
@Override | |||
@@ -49,14 +72,17 @@ public class EsProcessMonitor extends AbstractProcessMonitor { | |||
return true; | |||
} | |||
boolean flag = false; | |||
try { | |||
boolean flag = checkOperational(); | |||
if (flag) { | |||
nodeOperational.set(true); | |||
} | |||
flag = checkOperational(); | |||
} catch (InterruptedException e) { | |||
LOG.trace("Interrupted while checking ES node is operational", e); | |||
Thread.currentThread().interrupt(); | |||
} finally { | |||
if (flag) { | |||
transportClient.set(null); | |||
nodeOperational.set(true); | |||
} | |||
} | |||
return nodeOperational.get(); | |||
} | |||
@@ -73,30 +99,97 @@ public class EsProcessMonitor extends AbstractProcessMonitor { | |||
status = checkStatus(); | |||
} | |||
} while (!nodeUp.get() && i < WAIT_FOR_UP_TIMEOUT); | |||
return status == Status.YELLOW || status == Status.GREEN; | |||
return status == YELLOW || status == GREEN; | |||
} | |||
static class MinimalTransportClient extends TransportClient { | |||
MinimalTransportClient(Settings settings) { | |||
super(settings, Settings.EMPTY, unmodifiableList(singletonList(Netty4Plugin.class))); | |||
} | |||
@Override | |||
public void close() { | |||
super.close(); | |||
if (NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings) == false | |||
|| NetworkModule.TRANSPORT_TYPE_SETTING.get(settings).equals(Netty4Plugin.NETTY_TRANSPORT_NAME)) { | |||
try { | |||
GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS); | |||
} catch (InterruptedException e) { | |||
Thread.currentThread().interrupt(); | |||
} | |||
try { | |||
ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS); | |||
} catch (InterruptedException e) { | |||
Thread.currentThread().interrupt(); | |||
} | |||
} | |||
} | |||
} | |||
private Status checkStatus() { | |||
try { | |||
URLConnection urlConnection = healthCheckURL.openConnection(); | |||
urlConnection.connect(); | |||
String response = IOUtils.toString(urlConnection.getInputStream()); | |||
if (response.contains("\"status\":\"green\"")) { | |||
return Status.GREEN; | |||
} else if (response.contains("\"status\":\"yellow\"")) { | |||
return Status.YELLOW; | |||
} else if (response.contains("\"status\":\"red\"")) { | |||
return Status.RED; | |||
ClusterHealthResponse response = getTransportClient().admin().cluster() | |||
.health(new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.YELLOW).timeout(timeValueSeconds(30))) | |||
.actionGet(); | |||
if (response.getStatus() == ClusterHealthStatus.GREEN) { | |||
return GREEN; | |||
} | |||
if (response.getStatus() == ClusterHealthStatus.YELLOW) { | |||
return YELLOW; | |||
} | |||
return Status.KO; | |||
} catch (ConnectException e) { | |||
return Status.CONNECTION_REFUSED; | |||
} catch (IOException e) { | |||
LOG.error("Unexpected error occurred while checking ES node status using WebService API", e); | |||
return Status.KO; | |||
if (response.getStatus() == ClusterHealthStatus.RED) { | |||
return RED; | |||
} | |||
return KO; | |||
} catch (NoNodeAvailableException e) { | |||
return CONNECTION_REFUSED; | |||
} catch (Exception e) { | |||
LOG.error("Failed to check status", e); | |||
return KO; | |||
} | |||
} | |||
private TransportClient getTransportClient() { | |||
TransportClient res = this.transportClient.get(); | |||
if (res == null) { | |||
res = buildTransportClient(); | |||
if (this.transportClient.compareAndSet(null, res)) { | |||
return res; | |||
} | |||
return this.transportClient.get(); | |||
} | |||
return res; | |||
} | |||
private TransportClient buildTransportClient() { | |||
org.elasticsearch.common.settings.Settings.Builder esSettings = org.elasticsearch.common.settings.Settings.builder(); | |||
// mandatory property defined by bootstrap process | |||
esSettings.put("cluster.name", esCommand.getClusterName()); | |||
TransportClient nativeClient = new MinimalTransportClient(esSettings.build()); | |||
HostAndPort host = HostAndPort.fromParts(esCommand.getHost(), esCommand.getPort()); | |||
addHostToClient(host, nativeClient); | |||
if (LOG.isDebugEnabled()) { | |||
LOG.debug("Connected to Elasticsearch node: [{}]", displayedAddresses(nativeClient)); | |||
} | |||
return nativeClient; | |||
} | |||
private static void addHostToClient(HostAndPort host, TransportClient client) { | |||
try { | |||
client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host.getHostText()), host.getPortOrDefault(9001))); | |||
} catch (UnknownHostException e) { | |||
throw new IllegalStateException("Can not resolve host [" + host + "]", e); | |||
} | |||
} | |||
private static String displayedAddresses(TransportClient nativeClient) { | |||
return nativeClient.transportAddresses().stream().map(TransportAddress::toString).collect(Collectors.joining(", ")); | |||
} | |||
enum Status { | |||
CONNECTION_REFUSED, KO, RED, YELLOW, GREEN | |||
} |
@@ -71,7 +71,7 @@ public class ProcessLauncherImpl implements ProcessLauncher { | |||
process = processBuilder.start(); | |||
return new EsProcessMonitor(process, esCommand.getUrl()); | |||
return new EsProcessMonitor(process, esCommand); | |||
} catch (Exception e) { | |||
// just in case | |||
if (process != null) { |
@@ -34,6 +34,7 @@ import org.junit.rules.ExpectedException; | |||
import org.junit.rules.TestRule; | |||
import org.junit.rules.Timeout; | |||
import org.mockito.Mockito; | |||
import org.sonar.application.config.AppSettings; | |||
import org.sonar.application.config.TestAppSettings; | |||
import org.sonar.application.process.AbstractCommand; | |||
import org.sonar.application.process.CommandFactory; | |||
@@ -309,7 +310,7 @@ public class SchedulerImplTest { | |||
private static class TestCommandFactory implements CommandFactory { | |||
@Override | |||
public EsCommand createEsCommand() { | |||
public EsCommand createEsCommand(AppSettings settings) { | |||
return ES_COMMAND; | |||
} | |||
@@ -19,10 +19,6 @@ | |||
<artifactId>sonar-process</artifactId> | |||
<version>${project.version}</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>net.java.dev.jna</groupId> | |||
<artifactId>jna</artifactId> | |||
</dependency> | |||
<dependency> | |||
<groupId>com.google.code.findbugs</groupId> | |||
<artifactId>jsr305</artifactId> |
@@ -61,7 +61,7 @@ public class EsSettings implements EsSettingsMBean { | |||
@Override | |||
public int getHttpPort() { | |||
return props.valueAsInt(ProcessProperties.SEARCH_HTTP_PORT, 9010); | |||
return props.valueAsInt(ProcessProperties.SEARCH_HTTP_PORT, -1); | |||
} | |||
@Override | |||
@@ -125,15 +125,16 @@ public class EsSettings implements EsSettingsMBean { | |||
int httpPort = getHttpPort(); | |||
if (httpPort < 0) { | |||
// standard configuration | |||
httpPort = 9010; | |||
builder.put("http.enabled", String.valueOf(false)); | |||
} else { | |||
LOGGER.warn("Elasticsearch HTTP connector is enabled on port {}. MUST NOT BE USED FOR PRODUCTION", httpPort); | |||
// see https://github.com/lmenezes/elasticsearch-kopf/issues/195 | |||
builder.put("http.cors.enabled", String.valueOf(true)); | |||
builder.put("http.cors.allow-origin", "*"); | |||
builder.put("http.enabled", String.valueOf(true)); | |||
builder.put("http.host", host.getHostAddress()); | |||
builder.put("http.port", String.valueOf(httpPort)); | |||
} | |||
// see https://github.com/lmenezes/elasticsearch-kopf/issues/195 | |||
builder.put("http.cors.enabled", String.valueOf(true)); | |||
builder.put("http.cors.allow-origin", "*"); | |||
builder.put("http.enabled", String.valueOf(true)); | |||
builder.put("http.host", host.getHostAddress()); | |||
builder.put("http.port", String.valueOf(httpPort)); | |||
} | |||
private InetAddress readHost() { |
@@ -149,19 +149,15 @@ | |||
<artifactId>commons-dbcp</artifactId> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.elasticsearch</groupId> | |||
<artifactId>elasticsearch</artifactId> | |||
<groupId>org.apache.httpcomponents</groupId> | |||
<artifactId>httpclient</artifactId> | |||
<version>4.5.2</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.elasticsearch.client</groupId> | |||
<artifactId>transport</artifactId> | |||
<version>${elasticsearch.version}</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.apache.httpcomponents</groupId> | |||
<artifactId>httpclient</artifactId> | |||
<version>4.5.2</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.elasticsearch.test</groupId> | |||
<artifactId>framework</artifactId> | |||
@@ -175,10 +171,6 @@ | |||
</exclusion> | |||
</exclusions> | |||
</dependency> | |||
<dependency> | |||
<groupId>net.java.dev.jna</groupId> | |||
<artifactId>jna</artifactId> | |||
</dependency> | |||
<dependency> | |||
<groupId>com.google.protobuf</groupId> | |||
<artifactId>protobuf-java</artifactId> |
@@ -36,17 +36,6 @@ | |||
<scope>runtime</scope> | |||
</dependencySet> | |||
<dependencySet> | |||
<outputDirectory>lib/common</outputDirectory> | |||
<useTransitiveFiltering>true</useTransitiveFiltering> | |||
<useProjectArtifact>false</useProjectArtifact> | |||
<includes> | |||
<include>org.elasticsearch:elasticsearch</include> | |||
<include>net.java.dev.jna:jna</include> | |||
</includes> | |||
<scope>provided</scope> | |||
</dependencySet> | |||
<dependencySet> | |||
<outputDirectory>lib/search</outputDirectory> | |||
<useProjectArtifact>false</useProjectArtifact> |
@@ -30,17 +30,18 @@ | |||
<artifactId>sonar-process-monitor</artifactId> | |||
<version>${project.version}</version> | |||
</dependency> | |||
<!--must declare this dependency of sonar-process-monitor here, again,--> | |||
<!--to allow copying it and its dependencies into lib/common--> | |||
<dependency> | |||
<groupId>org.elasticsearch.client</groupId> | |||
<artifactId>transport</artifactId> | |||
</dependency> | |||
<dependency> | |||
<groupId>com.google.code.findbugs</groupId> | |||
<artifactId>jsr305</artifactId> | |||
<scope>provided</scope> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.elasticsearch</groupId> | |||
<artifactId>elasticsearch</artifactId> | |||
<scope>provided</scope> | |||
</dependency> | |||
<dependency> | |||
<groupId>${project.groupId}</groupId> | |||
@@ -206,6 +207,23 @@ | |||
</goals> | |||
<configuration> | |||
<keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope> | |||
<artifactSet> | |||
<!--excluding some transitive dependencies which are not necessary to the main process to create--> | |||
<!--a smaller jar and use less memory--> | |||
<excludes> | |||
<exclude>org.apache.lucene:lucene-analyzers-common</exclude> | |||
<exclude>org.apache.lucene:lucene-backward-codecs</exclude> | |||
<exclude>org.apache.lucene:lucene-grouping</exclude> | |||
<exclude>org.apache.lucene:lucene-memory</exclude> | |||
<exclude>org.apache.lucene:lucene-misc</exclude> | |||
<exclude>org.apache.lucene:lucene-spatial-extras</exclude> | |||
<exclude>org.apache.lucene:lucene-spatial3d</exclude> | |||
<exclude>org.elasticsearch.plugin:reindex-client</exclude> | |||
<exclude>org.elasticsearch.plugin:lang-mustache-client</exclude> | |||
<exclude>org.elasticsearch.plugin:percolator-client</exclude> | |||
<exclude>org.elasticsearch.plugin:transport-netty3-client</exclude> | |||
</excludes> | |||
</artifactSet> | |||
</configuration> | |||
</execution> | |||
</executions> | |||
@@ -242,8 +260,8 @@ | |||
<configuration> | |||
<rules> | |||
<requireFilesSize> | |||
<minsize>178000000</minsize> | |||
<maxsize>186000000</maxsize> | |||
<minsize>202000000</minsize> | |||
<maxsize>210000000</maxsize> | |||
<files> | |||
<file>${project.build.directory}/sonarqube-${project.version}.zip</file> | |||
</files> |