package org.elasticsearch.client:transport and most of its dependencies into sonar-application.jar; do not open Elasticsearch's http port by default
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>net.java.dev.jna</groupId>
- <artifactId>jna</artifactId>
- <version>4.1.0</version>
- </dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast</artifactId>
<version>${hazelcast.version}</version>
</dependency>
<dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>transport</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>transport</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>2.6.2</version>
+ </dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
private void tryToStartEs() {
SQProcess process = processesById.get(ProcessId.ELASTICSEARCH);
if (process != null) {
- tryToStartEsProcess(process, commandFactory::createEsCommand);
+ tryToStartEsProcess(process, () -> commandFactory.createEsCommand(settings));
}
}
*/
package org.sonar.application.process;
+import org.sonar.application.config.AppSettings;
+
public interface CommandFactory {
- EsCommand createEsCommand();
+ EsCommand createEsCommand(AppSettings settings);
JavaCommand createWebCommand(boolean leader);
}
@Override
- public EsCommand createEsCommand() {
- File homeDir = settings.getProps().nonNullValueAsFile(ProcessProperties.PATH_HOME);
+ public EsCommand createEsCommand(AppSettings settings) {
+ File homeDir = this.settings.getProps().nonNullValueAsFile(ProcessProperties.PATH_HOME);
File executable = new File(homeDir, getExecutable());
if (!executable.exists()) {
throw new IllegalStateException("Cannot find elasticsearch binary");
}
- Map<String, String> settingsMap = new EsSettings(settings.getProps()).build();
+ Map<String, String> settingsMap = new EsSettings(this.settings.getProps()).build();
EsCommand res = new EsCommand(ProcessId.ELASTICSEARCH)
.setWorkDir(executable.getParentFile().getParentFile())
.setExecutable(executable)
- .setArguments(settings.getProps().rawProperties())
+ .setArguments(this.settings.getProps().rawProperties())
+ .setClusterName(settingsMap.get("cluster.name"))
+ .setHost(settingsMap.get("network.host"))
// TODO add argument to specify log4j configuration file
// TODO add argument to specify yaml configuration file
- .setUrl("http://" + settingsMap.get("http.host") + ":" + settingsMap.get("http.port"));
+ .setPort(Integer.valueOf(settingsMap.get("transport.tcp.port")));
settingsMap.entrySet().stream()
.filter(entry -> !"path.home".equals(entry.getKey()))
public class EsCommand extends AbstractCommand<EsCommand> {
private File executable;
- private String url;
+ private String clusterName;
+ private String host;
+ private int port;
private List<String> esOptions = new ArrayList<>();
public EsCommand(ProcessId id) {
return this;
}
- public String getUrl() {
- return url;
+ public String getClusterName() {
+ return clusterName;
}
- public EsCommand setUrl(String url) {
- this.url = url;
+ public EsCommand setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ return this;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public EsCommand setHost(String host) {
+ this.host = host;
+ return this;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public EsCommand setPort(int port) {
+ this.port = port;
return this;
}
*/
package org.sonar.application.process;
-import java.io.IOException;
-import java.net.ConnectException;
+import com.google.common.net.HostAndPort;
+import io.netty.util.ThreadDeathWatcher;
+import io.netty.util.concurrent.GlobalEventExecutor;
+import java.net.InetAddress;
import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLConnection;
+import java.net.UnknownHostException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.io.IOUtils;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.transport.Netty4Plugin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.sonar.application.process.EsProcessMonitor.Status.CONNECTION_REFUSED;
+import static org.sonar.application.process.EsProcessMonitor.Status.GREEN;
+import static org.sonar.application.process.EsProcessMonitor.Status.KO;
+import static org.sonar.application.process.EsProcessMonitor.Status.RED;
+import static org.sonar.application.process.EsProcessMonitor.Status.YELLOW;
+
public class EsProcessMonitor extends AbstractProcessMonitor {
private static final Logger LOG = LoggerFactory.getLogger(EsProcessMonitor.class);
private static final int WAIT_FOR_UP_DELAY_IN_MILLIS = 100;
private final AtomicBoolean nodeUp = new AtomicBoolean(false);
private final AtomicBoolean nodeOperational = new AtomicBoolean(false);
- private final URL healthCheckURL;
+ private final EsCommand esCommand;
+ private AtomicReference<TransportClient> transportClient = new AtomicReference<>(null);
- public EsProcessMonitor(Process process, String url) throws MalformedURLException {
+ public EsProcessMonitor(Process process, EsCommand esCommand) throws MalformedURLException {
super(process);
- this.healthCheckURL = new URL(url + "/_cluster/health?wait_for_status=yellow&timeout=30s");
+ this.esCommand = esCommand;
}
@Override
return true;
}
+ boolean flag = false;
try {
- boolean flag = checkOperational();
- if (flag) {
- nodeOperational.set(true);
- }
+ flag = checkOperational();
} catch (InterruptedException e) {
LOG.trace("Interrupted while checking ES node is operational", e);
Thread.currentThread().interrupt();
+ } finally {
+ if (flag) {
+ transportClient.set(null);
+ nodeOperational.set(true);
+ }
}
return nodeOperational.get();
}
status = checkStatus();
}
} while (!nodeUp.get() && i < WAIT_FOR_UP_TIMEOUT);
- return status == Status.YELLOW || status == Status.GREEN;
+ return status == YELLOW || status == GREEN;
+ }
+
+ static class MinimalTransportClient extends TransportClient {
+
+ MinimalTransportClient(Settings settings) {
+ super(settings, Settings.EMPTY, unmodifiableList(singletonList(Netty4Plugin.class)));
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ if (NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings) == false
+ || NetworkModule.TRANSPORT_TYPE_SETTING.get(settings).equals(Netty4Plugin.NETTY_TRANSPORT_NAME)) {
+ try {
+ GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ try {
+ ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
}
private Status checkStatus() {
try {
- URLConnection urlConnection = healthCheckURL.openConnection();
- urlConnection.connect();
- String response = IOUtils.toString(urlConnection.getInputStream());
- if (response.contains("\"status\":\"green\"")) {
- return Status.GREEN;
- } else if (response.contains("\"status\":\"yellow\"")) {
- return Status.YELLOW;
- } else if (response.contains("\"status\":\"red\"")) {
- return Status.RED;
+ ClusterHealthResponse response = getTransportClient().admin().cluster()
+ .health(new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.YELLOW).timeout(timeValueSeconds(30)))
+ .actionGet();
+ if (response.getStatus() == ClusterHealthStatus.GREEN) {
+ return GREEN;
+ }
+ if (response.getStatus() == ClusterHealthStatus.YELLOW) {
+ return YELLOW;
}
- return Status.KO;
- } catch (ConnectException e) {
- return Status.CONNECTION_REFUSED;
- } catch (IOException e) {
- LOG.error("Unexpected error occurred while checking ES node status using WebService API", e);
- return Status.KO;
+ if (response.getStatus() == ClusterHealthStatus.RED) {
+ return RED;
+ }
+ return KO;
+ } catch (NoNodeAvailableException e) {
+ return CONNECTION_REFUSED;
+ } catch (Exception e) {
+ LOG.error("Failed to check status", e);
+ return KO;
+ }
+ }
+
+ private TransportClient getTransportClient() {
+ TransportClient res = this.transportClient.get();
+ if (res == null) {
+ res = buildTransportClient();
+ if (this.transportClient.compareAndSet(null, res)) {
+ return res;
+ }
+ return this.transportClient.get();
+ }
+ return res;
+ }
+
+ private TransportClient buildTransportClient() {
+ org.elasticsearch.common.settings.Settings.Builder esSettings = org.elasticsearch.common.settings.Settings.builder();
+
+ // mandatory property defined by bootstrap process
+ esSettings.put("cluster.name", esCommand.getClusterName());
+
+ TransportClient nativeClient = new MinimalTransportClient(esSettings.build());
+ HostAndPort host = HostAndPort.fromParts(esCommand.getHost(), esCommand.getPort());
+ addHostToClient(host, nativeClient);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connected to Elasticsearch node: [{}]", displayedAddresses(nativeClient));
+ }
+ return nativeClient;
+ }
+
+ private static void addHostToClient(HostAndPort host, TransportClient client) {
+ try {
+ client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host.getHostText()), host.getPortOrDefault(9001)));
+ } catch (UnknownHostException e) {
+ throw new IllegalStateException("Can not resolve host [" + host + "]", e);
}
}
+ private static String displayedAddresses(TransportClient nativeClient) {
+ return nativeClient.transportAddresses().stream().map(TransportAddress::toString).collect(Collectors.joining(", "));
+ }
+
enum Status {
CONNECTION_REFUSED, KO, RED, YELLOW, GREEN
}
process = processBuilder.start();
- return new EsProcessMonitor(process, esCommand.getUrl());
+ return new EsProcessMonitor(process, esCommand);
} catch (Exception e) {
// just in case
if (process != null) {
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
import org.mockito.Mockito;
+import org.sonar.application.config.AppSettings;
import org.sonar.application.config.TestAppSettings;
import org.sonar.application.process.AbstractCommand;
import org.sonar.application.process.CommandFactory;
private static class TestCommandFactory implements CommandFactory {
@Override
- public EsCommand createEsCommand() {
+ public EsCommand createEsCommand(AppSettings settings) {
return ES_COMMAND;
}
<artifactId>sonar-process</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>net.java.dev.jna</groupId>
- <artifactId>jna</artifactId>
- </dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
@Override
public int getHttpPort() {
- return props.valueAsInt(ProcessProperties.SEARCH_HTTP_PORT, 9010);
+ return props.valueAsInt(ProcessProperties.SEARCH_HTTP_PORT, -1);
}
@Override
int httpPort = getHttpPort();
if (httpPort < 0) {
// standard configuration
- httpPort = 9010;
+ builder.put("http.enabled", String.valueOf(false));
+ } else {
+ LOGGER.warn("Elasticsearch HTTP connector is enabled on port {}. MUST NOT BE USED FOR PRODUCTION", httpPort);
+ // see https://github.com/lmenezes/elasticsearch-kopf/issues/195
+ builder.put("http.cors.enabled", String.valueOf(true));
+ builder.put("http.cors.allow-origin", "*");
+ builder.put("http.enabled", String.valueOf(true));
+ builder.put("http.host", host.getHostAddress());
+ builder.put("http.port", String.valueOf(httpPort));
}
-
- // see https://github.com/lmenezes/elasticsearch-kopf/issues/195
- builder.put("http.cors.enabled", String.valueOf(true));
- builder.put("http.cors.allow-origin", "*");
- builder.put("http.enabled", String.valueOf(true));
- builder.put("http.host", host.getHostAddress());
- builder.put("http.port", String.valueOf(httpPort));
}
private InetAddress readHost() {
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.2</version>
- </dependency>
<dependency>
<groupId>org.elasticsearch.test</groupId>
<artifactId>framework</artifactId>
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>net.java.dev.jna</groupId>
- <artifactId>jna</artifactId>
- </dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<scope>runtime</scope>
</dependencySet>
- <dependencySet>
- <outputDirectory>lib/common</outputDirectory>
- <useTransitiveFiltering>true</useTransitiveFiltering>
- <useProjectArtifact>false</useProjectArtifact>
- <includes>
- <include>org.elasticsearch:elasticsearch</include>
- <include>net.java.dev.jna:jna</include>
- </includes>
- <scope>provided</scope>
- </dependencySet>
-
<dependencySet>
<outputDirectory>lib/search</outputDirectory>
<useProjectArtifact>false</useProjectArtifact>
<artifactId>sonar-process-monitor</artifactId>
<version>${project.version}</version>
</dependency>
+ <!--must declare this dependency of sonar-process-monitor here, again,-->
+ <!--to allow copying it and its dependencies into lib/common-->
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>transport</artifactId>
+ </dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>${project.groupId}</groupId>
</goals>
<configuration>
<keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope>
+ <artifactSet>
+ <!--excluding some transitive dependencies which are not necessary to the main process to create-->
+ <!--a smaller jar and use less memory-->
+ <excludes>
+ <exclude>org.apache.lucene:lucene-analyzers-common</exclude>
+ <exclude>org.apache.lucene:lucene-backward-codecs</exclude>
+ <exclude>org.apache.lucene:lucene-grouping</exclude>
+ <exclude>org.apache.lucene:lucene-memory</exclude>
+ <exclude>org.apache.lucene:lucene-misc</exclude>
+ <exclude>org.apache.lucene:lucene-spatial-extras</exclude>
+ <exclude>org.apache.lucene:lucene-spatial3d</exclude>
+ <exclude>org.elasticsearch.plugin:reindex-client</exclude>
+ <exclude>org.elasticsearch.plugin:lang-mustache-client</exclude>
+ <exclude>org.elasticsearch.plugin:percolator-client</exclude>
+ <exclude>org.elasticsearch.plugin:transport-netty3-client</exclude>
+ </excludes>
+ </artifactSet>
</configuration>
</execution>
</executions>
<configuration>
<rules>
<requireFilesSize>
- <minsize>178000000</minsize>
- <maxsize>186000000</maxsize>
+ <minsize>202000000</minsize>
+ <maxsize>210000000</maxsize>
<files>
<file>${project.build.directory}/sonarqube-${project.version}.zip</file>
</files>
projects / sonarqube.git / commitdiff