diff options
author | Jacek <jacek.poreda@sonarsource.com> | 2020-01-09 09:26:53 +0100 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2020-11-05 20:06:21 +0000 |
commit | f4751bd13509f8d325d17cb4cf4ed9d85025f65f (patch) | |
tree | 369137df20a5df287bf77cdb7dcf49888f282f31 | |
parent | 8cdee7d30f96e87b8bb7ec55fdfd8101ab717dfd (diff) | |
download | sonarqube-f4751bd13509f8d325d17cb4cf4ed9d85025f65f.tar.gz sonarqube-f4751bd13509f8d325d17cb4cf4ed9d85025f65f.zip |
SONAR-12686 upgrade es client to 7.9.3 and move to HTTP
- add should minimum match eq 1 to user index queries
ES 7.X changed behaviour in case filter query with bool it defaults to '0'
https://www.elastic.co/guide/en/elasticsearch/reference/7.x/breaking-changes-7.0.html#_the_filter_context_has_been_removed
- fix issue index routing param
ES 7.X helped discover this bug as new setting has been auto configured which is 'index.number_of_routing_shards'.
This has changed how documents are distributed across shards depending on how many shards the index has.
Without that change issues docs has been incorrectly routed to the same shard hash as projects and it worked no matter what routing key you used projectUuid or auth_projectUuid.
- update ngram and edge_ngram names to match with es 7.x
nGram and edgeNgram has been deprecated in favour of ngram and edge_ngram
https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#deprecated-ngram-edgengram-token-filter-cannot-be-used
- remove `_all : enabled` usage from UT
This field was already deprecated in 6.X, now it has been removed.
https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#all-meta-field-removed
- add Elasticsearch High Level REST client dependency
- use sonar.search.port for ES HTTP
- main process use ES Rest client to check ES status
- sonar.cluster.search.hosts has HTTP ports on APP nodes
also sonar.search.port and sonar.search.host MUST be configured on each Search node with the host and HTTP port of the current node
- use Elasticsearch high level rest client
- use in EsTester
- use as primary es client
- use indices api to get all indices name instead of cluster api
- use cluster health api to check cluster state
- support raw requests for 'nodes/_stats' and '_cluster/stats'
- support raw requests for 'indices/_stats'
- leave netty4plugin as testCompile dependency it is used in UTs
- all ES non-test calls go through EsClient class
- add rest client ES profiling
123 files changed, 2599 insertions, 3706 deletions
diff --git a/build.gradle b/build.gradle index 9185f1473d3..a7e1f05d425 100644 --- a/build.gradle +++ b/build.gradle @@ -342,14 +342,12 @@ subprojects { entry 'jetty-server' entry 'jetty-servlet' } - dependency('org.elasticsearch.client:transport:6.8.4') { - exclude 'org.elasticsearch.plugin:lang-mustache-client' + dependency('org.elasticsearch.client:elasticsearch-rest-high-level-client:7.9.3') { exclude 'commons-logging:commons-logging' - exclude 'org.elasticsearch.plugin:reindex-client' - exclude 'org.elasticsearch.plugin:rank-eval-client' } + dependency 'org.elasticsearch.plugin:transport-netty4-client:7.9.3' dependency 'org.elasticsearch:mocksocket:1.0' - dependency 'org.codelibs.elasticsearch.module:analysis-common:6.8.4' + dependency 'org.codelibs.elasticsearch.module:analysis-common:7.9.3' dependency 'org.eclipse.jgit:org.eclipse.jgit:5.9.0.202009080501-r' dependency 'org.tmatesoft.svnkit:svnkit:1.10.1' dependency 'org.hamcrest:hamcrest-all:1.3' diff --git a/server/sonar-main/build.gradle b/server/sonar-main/build.gradle index cb67a49fe04..2e11dd66e75 100644 --- a/server/sonar-main/build.gradle +++ b/server/sonar-main/build.gradle @@ -15,13 +15,11 @@ dependencies { compile 'com.hazelcast:hazelcast' compile 'commons-io:commons-io' compile 'commons-lang:commons-lang' - compile 'io.netty:netty-common' compile 'org.apache.logging.log4j:log4j-to-slf4j' compile 'org.apache.logging.log4j:log4j-api' - compile 'org.elasticsearch.client:transport' + compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client' compile 'org.elasticsearch:elasticsearch' compile 'org.elasticsearch:elasticsearch-core' - compile 'org.elasticsearch.plugin:transport-netty4-client' compile 'org.slf4j:slf4j-api' compile 'org.yaml:snakeyaml' @@ -36,5 +34,7 @@ dependencies { testCompile 'org.assertj:assertj-core' testCompile 'org.awaitility:awaitility' testCompile 'org.mockito:mockito-core' + testCompile 'com.squareup.okhttp3:mockwebserver' + testCompile 'commons-logging:commons-logging:1.1.1' testCompile project(':sonar-testing-harness') } diff --git a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java index 6a70a5ce795..b486faadbd8 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java +++ b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java @@ -19,6 +19,7 @@ */ package org.sonar.application; +import com.google.common.collect.ImmutableSet; import com.google.common.net.HostAndPort; import java.util.Arrays; import java.util.Set; @@ -31,16 +32,18 @@ import org.sonar.application.es.EsConnector; import org.sonar.application.es.EsConnectorImpl; import org.sonar.process.ProcessId; import org.sonar.process.Props; +import org.sonar.process.cluster.NodeType; import org.sonar.process.cluster.hz.HazelcastMember; import org.sonar.process.cluster.hz.HazelcastMemberBuilder; import static java.util.Arrays.asList; import static org.sonar.process.ProcessProperties.Property.CLUSTER_HZ_HOSTS; -import static org.sonar.process.ProcessProperties.Property.CLUSTER_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HOST; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HZ_PORT; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_HOSTS; +import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST; +import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT; public class AppStateFactory { private final AppSettings settings; @@ -70,10 +73,17 @@ public class AppStateFactory { } private static EsConnector createEsConnector(Props props) { + NodeType nodeType = ClusterSettings.toNodeType(props); + if (nodeType == NodeType.SEARCH) { + String host = props.nonNullValue(SEARCH_HOST.getKey()); + String port = props.nonNullValue(SEARCH_PORT.getKey()); + return new EsConnectorImpl(ImmutableSet.of(HostAndPort.fromParts(host, Integer.valueOf(port)))); + } + String searchHosts = props.nonNullValue(CLUSTER_SEARCH_HOSTS.getKey()); Set<HostAndPort> hostAndPorts = Arrays.stream(searchHosts.split(",")) .map(HostAndPort::fromString) .collect(Collectors.toSet()); - return new EsConnectorImpl(props.nonNullValue(CLUSTER_NAME.getKey()), hostAndPorts); + return new EsConnectorImpl(hostAndPorts); } } diff --git a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java index e0c4aab97f2..d6a23e5fe2f 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java @@ -97,7 +97,7 @@ public class ProcessLauncherImpl implements ProcessLauncher { try { if (processId == ProcessId.ELASTICSEARCH) { checkArgument(esInstallation != null, "Incorrect configuration EsInstallation is null"); - EsConnectorImpl esConnector = new EsConnectorImpl(esInstallation.getClusterName(), singleton(HostAndPort.fromParts(esInstallation.getHost(), esInstallation.getPort()))); + EsConnectorImpl esConnector = new EsConnectorImpl(singleton(HostAndPort.fromParts(esInstallation.getHost(), esInstallation.getHttpPort()))); return new EsManagedProcess(process, processId, esConnector); } else { ProcessCommands commands = allProcessesCommands.createAfterClean(processId.getIpcIndex()); diff --git a/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java b/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java index 5efe6f0b999..6eec7ff5746 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java @@ -200,8 +200,9 @@ public class ClusterAppStateImpl implements ClusterAppState { } private boolean isElasticSearchAvailable() { - ClusterHealthStatus clusterHealthStatus = esConnector.getClusterHealthStatus(); - return clusterHealthStatus.equals(ClusterHealthStatus.GREEN) || clusterHealthStatus.equals(ClusterHealthStatus.YELLOW); + return esConnector.getClusterHealthStatus() + .filter(t -> ClusterHealthStatus.GREEN.equals(t) || ClusterHealthStatus.YELLOW.equals(t)) + .isPresent(); } private class OperationalProcessListener implements EntryListener<ClusterProcess, Boolean> { diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java index 37cd0c6c31e..353780521d3 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java @@ -143,9 +143,8 @@ public class CommandFactoryImpl implements CommandFactory { .addFromMandatoryProperty(props, SEARCH_JAVA_OPTS.getKey()) .addFromMandatoryProperty(props, SEARCH_JAVA_ADDITIONAL_OPTS.getKey())) .setEsYmlSettings(new EsYmlSettings(settingsMap)) - .setClusterName(settingsMap.get("cluster.name")) - .setHost(settingsMap.get("network.host")) - .setPort(Integer.parseInt(settingsMap.get("transport.port"))); + .setHost(settingsMap.get("http.host")) + .setHttpPort(Integer.parseInt(settingsMap.get("http.port"))); return esInstallation; } diff --git a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java index 85d7078381c..997e68a5f50 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java +++ b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java @@ -138,7 +138,7 @@ public class ClusterSettings implements Consumer<Props> { return addressAndPort; } - private static NodeType toNodeType(Props props) { + public static NodeType toNodeType(Props props) { String nodeTypeValue = requireValue(props, CLUSTER_NODE_TYPE); if (!NodeType.isValid(nodeTypeValue)) { throw new MessageException(format("Invalid value for property %s: [%s], only [%s] are allowed", CLUSTER_NODE_TYPE.getKey(), nodeTypeValue, diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java index 6e289407904..78544b302a5 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java @@ -19,9 +19,11 @@ */ package org.sonar.application.es; +import java.util.Optional; import org.elasticsearch.cluster.health.ClusterHealthStatus; public interface EsConnector { - ClusterHealthStatus getClusterHealthStatus(); + Optional<ClusterHealthStatus> getClusterHealthStatus(); + void stop(); } diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java index 8252014f12d..bacc25c85b6 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java @@ -20,121 +20,83 @@ package org.sonar.application.es; import com.google.common.net.HostAndPort; -import io.netty.util.ThreadDeathWatcher; -import io.netty.util.concurrent.GlobalEventExecutor; -import java.net.InetAddress; -import java.net.UnknownHostException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Optional; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import org.apache.http.HttpHost; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.transport.Netty4Plugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static java.lang.String.format; -import static java.util.Collections.singletonList; -import static java.util.Collections.unmodifiableList; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; public class EsConnectorImpl implements EsConnector { private static final Logger LOG = LoggerFactory.getLogger(EsConnectorImpl.class); - private final AtomicReference<TransportClient> transportClient = new AtomicReference<>(null); - private final String clusterName; + private final AtomicReference<RestHighLevelClient> restClient = new AtomicReference<>(null); private final Set<HostAndPort> hostAndPorts; - public EsConnectorImpl(String clusterName, Set<HostAndPort> hostAndPorts) { - this.clusterName = clusterName; + public EsConnectorImpl(Set<HostAndPort> hostAndPorts) { this.hostAndPorts = hostAndPorts; } @Override - public ClusterHealthStatus getClusterHealthStatus() { - return getTransportClient().admin().cluster() - .health(new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.YELLOW).timeout(timeValueSeconds(30))) - .actionGet().getStatus(); + public Optional<ClusterHealthStatus> getClusterHealthStatus() { + try { + ClusterHealthResponse healthResponse = getRestHighLevelClient().cluster() + .health(new ClusterHealthRequest().waitForYellowStatus().timeout(timeValueSeconds(30)), RequestOptions.DEFAULT); + return Optional.of(healthResponse.getStatus()); + } catch (IOException e) { + LOG.trace("Failed to check health status ", e); + return Optional.empty(); + } } @Override public void stop() { - transportClient.set(null); - } - - private TransportClient getTransportClient() { - TransportClient res = this.transportClient.get(); - if (res == null) { - res = buildTransportClient(); - if (this.transportClient.compareAndSet(null, res)) { - return res; + RestHighLevelClient restHighLevelClient = restClient.get(); + if (restHighLevelClient != null) { + try { + restHighLevelClient.close(); + } catch (IOException e) { + LOG.warn("Error occurred while closing Rest Client", e); } - return this.transportClient.get(); } - return res; } - private TransportClient buildTransportClient() { - Settings.Builder esSettings = Settings.builder(); - - // mandatory property defined by bootstrap process - esSettings.put("cluster.name", clusterName); - - TransportClient nativeClient = new MinimalTransportClient(esSettings.build(), hostAndPorts); - if (LOG.isDebugEnabled()) { - LOG.debug("Connected to Elasticsearch node: [{}]", displayedAddresses(nativeClient)); + private RestHighLevelClient getRestHighLevelClient() { + RestHighLevelClient res = this.restClient.get(); + if (res != null) { + return res; } - return nativeClient; - } - private static String displayedAddresses(TransportClient nativeClient) { - return nativeClient.transportAddresses().stream().map(TransportAddress::toString).collect(Collectors.joining(", ")); + RestHighLevelClient restHighLevelClient = buildRestHighLevelClient(); + this.restClient.set(restHighLevelClient); + return restHighLevelClient; } - private static class MinimalTransportClient extends TransportClient { - - public MinimalTransportClient(Settings settings, Set<HostAndPort> hostAndPorts) { - super(settings, unmodifiableList(singletonList(Netty4Plugin.class))); + private RestHighLevelClient buildRestHighLevelClient() { + HttpHost[] httpHosts = hostAndPorts.stream() + .map(hostAndPort -> new HttpHost(hostAndPort.getHost(), hostAndPort.getPortOrDefault(9001))) + .toArray(HttpHost[]::new); - boolean connectedToOneHost = false; - for (HostAndPort hostAndPort : hostAndPorts) { - try { - addTransportAddress(new TransportAddress(InetAddress.getByName(hostAndPort.getHost()), hostAndPort.getPortOrDefault(9001))); - connectedToOneHost = true; - } catch (UnknownHostException e) { - LOG.debug("Can not resolve host [" + hostAndPort.getHost() + "]", e); - } - } - if (!connectedToOneHost) { - throw new IllegalStateException(format("Can not connect to one node from [%s]", - hostAndPorts.stream() - .map(h -> format("%s:%d", h.getHost(), h.getPortOrDefault(9001))) - .collect(Collectors.joining(",")))); - } - } - - @Override - public void close() { - super.close(); - if (!NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings) - || NetworkModule.TRANSPORT_TYPE_SETTING.get(settings).equals(Netty4Plugin.NETTY_TRANSPORT_NAME)) { - try { - GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - try { - ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } + if (LOG.isDebugEnabled()) { + String addresses = Arrays.stream(httpHosts) + .map(t -> t.getHostName() + ":" + t.getPort()) + .collect(Collectors.joining(", ")); + LOG.debug("Connected to Elasticsearch node: [{}]", addresses); } + return new RestHighLevelClient(RestClient.builder(httpHosts)); } + } diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java index 6d084d23cba..5657f094adb 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java @@ -52,7 +52,7 @@ public class EsInstallation { private Properties log4j2Properties; private String clusterName; private String host; - private int port; + private int httpPort; public EsInstallation(Props props) { File sqHomeDir = props.nonNullValueAsFile(PATH_HOME.getKey()); @@ -152,15 +152,6 @@ public class EsInstallation { return this; } - public String getClusterName() { - return clusterName; - } - - public EsInstallation setClusterName(String clusterName) { - this.clusterName = clusterName; - return this; - } - public String getHost() { return host; } @@ -170,12 +161,12 @@ public class EsInstallation { return this; } - public int getPort() { - return port; + public int getHttpPort() { + return httpPort; } - public EsInstallation setPort(int port) { - this.port = port; + public EsInstallation setHttpPort(int httpPort) { + this.httpPort = httpPort; return this; } } diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java index a852fdf4efc..1a7b658c70a 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java @@ -36,9 +36,9 @@ import static org.sonar.process.ProcessProperties.Property.CLUSTER_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_HOSTS; import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST; -import static org.sonar.process.ProcessProperties.Property.SEARCH_HTTP_PORT; import static org.sonar.process.ProcessProperties.Property.SEARCH_INITIAL_STATE_TIMEOUT; import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT; +import static org.sonar.process.ProcessProperties.Property.SEARCH_TRANSPORT_PORT; public class EsSettings { @@ -75,8 +75,8 @@ public class EsSettings { public Map<String, String> build() { Map<String, String> builder = new HashMap<>(); configureFileSystem(builder); - configureNetwork(builder); - configureCluster(builder); + InetAddress host = configureNetwork(builder); + configureCluster(builder, host); configureOthers(builder); return builder; } @@ -86,33 +86,27 @@ public class EsSettings { builder.put("path.logs", fileSystem.getLogDirectory().getAbsolutePath()); } - private void configureNetwork(Map<String, String> builder) { + private InetAddress configureNetwork(Map<String, String> builder) { InetAddress host = readHost(); - int port = Integer.parseInt(props.nonNullValue(SEARCH_PORT.getKey())); - LOGGER.info("Elasticsearch listening on {}:{}", host, port); + int httpPort = Integer.parseInt(props.nonNullValue(SEARCH_PORT.getKey())); + LOGGER.info("Elasticsearch listening on {}:{}", host, httpPort); - // FIXME no need to open TCP port unless running DCE - // TCP is used by main process to check ES is up => probably has to use HTTP now - builder.put("transport.port", valueOf(port)); - builder.put("transport.host", valueOf(host.getHostAddress())); - builder.put("network.host", valueOf(host.getHostAddress())); + // see https://github.com/lmenezes/elasticsearch-kopf/issues/195 + builder.put("http.cors.enabled", valueOf(true)); + builder.put("http.cors.allow-origin", "*"); + builder.put("http.host", host.getHostAddress()); + builder.put("http.port", valueOf(httpPort)); + builder.put("network.host", valueOf(host.getHostAddress())); // Elasticsearch sets the default value of TCP reuse address to true only on non-MSWindows machines, but why ? builder.put("network.tcp.reuse_address", valueOf(true)); - int httpPort = props.valueAsInt(SEARCH_HTTP_PORT.getKey(), -1); - if (httpPort < 0) { - // standard configuration - builder.put("http.enabled", valueOf(false)); - } else { - LOGGER.warn("Elasticsearch HTTP connector is enabled on port {}. MUST NOT BE USED FOR PRODUCTION", httpPort); - // see https://github.com/lmenezes/elasticsearch-kopf/issues/195 - builder.put("http.cors.enabled", valueOf(true)); - builder.put("http.cors.allow-origin", "*"); - builder.put("http.enabled", valueOf(true)); - builder.put("http.host", host.getHostAddress()); - builder.put("http.port", valueOf(httpPort)); - } + // FIXME remove definition of transport properties when Web and CE have moved to ES Rest client + int tcpPort = props.valueAsInt(SEARCH_TRANSPORT_PORT.getKey(), 9002); + builder.put("transport.port", valueOf(tcpPort)); + builder.put("transport.host", valueOf(host.getHostAddress())); + + return host; } private InetAddress readHost() { @@ -124,7 +118,7 @@ public class EsSettings { } } - private void configureCluster(Map<String, String> builder) { + private void configureCluster(Map<String, String> builder, InetAddress host) { // Default value in a standalone mode, not overridable String initialStateTimeOut = "30s"; @@ -132,6 +126,10 @@ public class EsSettings { if (clusterEnabled) { initialStateTimeOut = props.value(SEARCH_INITIAL_STATE_TIMEOUT.getKey(), "120s"); + int tcpPort = props.valueAsInt(SEARCH_TRANSPORT_PORT.getKey(), 9002); + builder.put("transport.port", valueOf(tcpPort)); + builder.put("transport.host", valueOf(host.getHostAddress())); + String hosts = props.value(CLUSTER_SEARCH_HOSTS.getKey(), ""); LOGGER.info("Elasticsearch cluster enabled. Connect to hosts [{}]", hosts); builder.put("discovery.seed_hosts", hosts); diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java index 6fe8f1acc35..36008dc3988 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java +++ b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java @@ -20,8 +20,9 @@ package org.sonar.application.process; import java.util.concurrent.atomic.AtomicBoolean; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.discovery.MasterNotDiscoveredException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.rest.RestStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.application.es.EsConnector; @@ -85,21 +86,14 @@ public class EsManagedProcess extends AbstractManagedProcess { private Status checkStatus() { try { - switch (esConnector.getClusterHealthStatus()) { - case GREEN: - return GREEN; - case YELLOW: - return YELLOW; - case RED: - return RED; - default: - return KO; - } - } catch (NoNodeAvailableException e) { - return CONNECTION_REFUSED; - } catch (MasterNotDiscoveredException e) { - if (firstMasterNotDiscoveredLog.getAndSet(false)) { - LOG.info("Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?"); + return esConnector.getClusterHealthStatus() + .map(EsManagedProcess::convert) + .orElse(CONNECTION_REFUSED); + } catch (ElasticsearchStatusException e) { + if (e.status() == RestStatus.SERVICE_UNAVAILABLE && e.getMessage().contains("type=master_not_discovered_exception")) { + if (firstMasterNotDiscoveredLog.getAndSet(false)) { + LOG.info("Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?"); + } } return KO; } catch (Exception e) { @@ -108,6 +102,19 @@ public class EsManagedProcess extends AbstractManagedProcess { } } + private static Status convert(ClusterHealthStatus clusterHealthStatus) { + switch (clusterHealthStatus) { + case GREEN: + return GREEN; + case YELLOW: + return YELLOW; + case RED: + return RED; + default: + return KO; + } + } + enum Status { CONNECTION_REFUSED, KO, RED, YELLOW, GREEN } diff --git a/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java b/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java index 1ae9c798b9a..48cbf0754e9 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java @@ -51,7 +51,7 @@ public class AppStateFactoryTest { settings.set(CLUSTER_NODE_HOST.getKey(), ip.get().getHostAddress()); settings.set(CLUSTER_HZ_HOSTS.getKey(), ip.get().getHostAddress()); settings.set(CLUSTER_NAME.getKey(), "foo"); - settings.set(CLUSTER_SEARCH_HOSTS.getKey(), "localhost:9001"); + settings.set(CLUSTER_SEARCH_HOSTS.getKey(), "localhost:9002"); AppState appState = underTest.create(); assertThat(appState).isInstanceOf(ClusterAppStateImpl.class); diff --git a/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java index 30360924a9a..9612aff58d5 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java @@ -200,8 +200,7 @@ public class ProcessLauncherImplTest { .setEsJvmOptions(mock(EsJvmOptions.class)) .setLog4j2Properties(new Properties()) .setHost("localhost") - .setPort(9001) - .setClusterName("sonarqube")); + .setHttpPort(9001)); return command; } @@ -212,8 +211,7 @@ public class ProcessLauncherImplTest { .set("sonar.path.data", this.temp.newFolder("data").getAbsolutePath()) .set("sonar.path.temp", tempFolder.getAbsolutePath()) .set("sonar.path.logs", this.temp.newFolder("logs").getAbsolutePath())) - .setClusterName("cluster") - .setPort(9001) + .setHttpPort(9001) .setHost("localhost") .setEsYmlSettings(new EsYmlSettings(new HashMap<>())) .setEsJvmOptions(new EsJvmOptions(new Props(new Properties()), tempFolder)) diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java index ee0537edc3a..bbe96d163b0 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java @@ -131,9 +131,8 @@ public class CommandFactoryImplTest { EsScriptCommand esCommand = (EsScriptCommand) command; EsInstallation esConfig = esCommand.getEsInstallation(); - assertThat(esConfig.getClusterName()).isEqualTo("sonarqube"); assertThat(esConfig.getHost()).isNotEmpty(); - assertThat(esConfig.getPort()).isEqualTo(9001); + assertThat(esConfig.getHttpPort()).isEqualTo(9001); assertThat(esConfig.getEsJvmOptions().getAll()) // enforced values .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8") @@ -167,9 +166,8 @@ public class CommandFactoryImplTest { JavaCommand<?> esCommand = (JavaCommand<?>) command; EsInstallation esConfig = esCommand.getEsInstallation(); - assertThat(esConfig.getClusterName()).isEqualTo("sonarqube"); assertThat(esConfig.getHost()).isNotEmpty(); - assertThat(esConfig.getPort()).isEqualTo(9001); + assertThat(esConfig.getHttpPort()).isEqualTo(9001); assertThat(esConfig.getEsJvmOptions().getAll()) // enforced values .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8") @@ -209,8 +207,7 @@ public class CommandFactoryImplTest { AbstractCommand esCommand = newFactory(props).createEsCommand(); EsInstallation esConfig = esCommand.getEsInstallation(); - assertThat(esConfig.getClusterName()).isEqualTo("foo"); - assertThat(esConfig.getPort()).isEqualTo(1234); + assertThat(esConfig.getHttpPort()).isEqualTo(1234); assertThat(esConfig.getEsJvmOptions().getAll()) // enforced values .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8") diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java new file mode 100644 index 00000000000..3bacc724278 --- /dev/null +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java @@ -0,0 +1,92 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.application.es; + +import com.google.common.collect.Sets; +import com.google.common.net.HostAndPort; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class EsConnectorImplTest { + + private static final String JSON_SUCCESS_RESPONSE = "{" + + " \"cluster_name\" : \"testcluster\"," + + " \"status\" : \"yellow\"," + + " \"timed_out\" : false," + + " \"number_of_nodes\" : 1," + + " \"number_of_data_nodes\" : 1," + + " \"active_primary_shards\" : 1," + + " \"active_shards\" : 1," + + " \"relocating_shards\" : 0," + + " \"initializing_shards\" : 0," + + " \"unassigned_shards\" : 1," + + " \"delayed_unassigned_shards\": 0," + + " \"number_of_pending_tasks\" : 0," + + " \"number_of_in_flight_fetch\": 0," + + " \"task_max_waiting_in_queue_millis\": 0," + + " \"active_shards_percent_as_number\": 50.0" + + "}"; + + private static final String JSON_ERROR_RESPONSE = "{" + + " \"error\" : \"i-have-a-bad-feelings-about-this\"" + + "}"; + + @Rule + public MockWebServer mockWebServer = new MockWebServer(); + + EsConnectorImpl underTest = new EsConnectorImpl(Sets.newHashSet(HostAndPort.fromParts(mockWebServer.getHostName(), mockWebServer.getPort()))); + + @After + public void after() { + underTest.stop(); + } + + @Test + public void should_rethrow_if_es_exception() { + mockServerResponse(500, JSON_ERROR_RESPONSE); + + assertThatThrownBy(() -> underTest.getClusterHealthStatus()) + .isInstanceOf(ElasticsearchStatusException.class); + } + + @Test + public void should_return_status() { + mockServerResponse(200, JSON_SUCCESS_RESPONSE); + + assertThat(underTest.getClusterHealthStatus()) + .hasValue(ClusterHealthStatus.YELLOW); + } + + private void mockServerResponse(int httpCode, String jsonResponse) { + mockWebServer.enqueue(new MockResponse() + .setResponseCode(httpCode) + .setBody(jsonResponse) + .setHeader("Content-Type", "application/json")); + } + +} diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java index cfc7800d52e..306f6a79332 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java @@ -56,7 +56,6 @@ import static org.sonar.process.ProcessProperties.Property.PATH_HOME; import static org.sonar.process.ProcessProperties.Property.PATH_LOGS; import static org.sonar.process.ProcessProperties.Property.PATH_TEMP; import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST; -import static org.sonar.process.ProcessProperties.Property.SEARCH_HTTP_PORT; import static org.sonar.process.ProcessProperties.Property.SEARCH_INITIAL_STATE_TIMEOUT; import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT; @@ -138,9 +137,14 @@ public class EsSettingsTest { EsSettings esSettings = new EsSettings(props, new EsInstallation(props), system); Map<String, String> generated = esSettings.build(); - assertThat(generated.get("transport.port")).isEqualTo("1234"); + + // FIXME transport.port and transport.host should not be set in standalone + assertThat(generated.get("transport.port")).isEqualTo("9002"); assertThat(generated.get("transport.host")).isEqualTo("127.0.0.1"); + assertThat(generated.get("http.port")).isEqualTo("1234"); + assertThat(generated.get("http.host")).isEqualTo("127.0.0.1"); + // no cluster, but cluster and node names are set though assertThat(generated.get("cluster.name")).isEqualTo("sonarqube"); assertThat(generated.get("node.name")).isEqualTo("sonarqube"); @@ -150,9 +154,6 @@ public class EsSettingsTest { assertThat(generated.get("path.home")).isNull(); assertThat(generated.get("path.conf")).isNull(); - // http is disabled for security reasons - assertThat(generated.get("http.enabled")).isEqualTo("false"); - assertThat(generated.get("discovery.seed_hosts")).isNull(); assertThat(generated.get("discovery.initial_state_timeout")).isEqualTo("30s"); @@ -274,39 +275,51 @@ public class EsSettingsTest { } @Test - public void enable_http_connector() throws Exception { - Props props = minProps(CLUSTER_DISABLED); - props.set(SEARCH_HTTP_PORT.getKey(), "9010"); + @UseDataProvider("clusterEnabledOrNot") + public void enable_http_connector_on_port_9001_by_default(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); - assertThat(settings.get("http.port")).isEqualTo("9010"); + assertThat(settings.get("http.port")).isEqualTo("9001"); assertThat(settings.get("http.host")).isEqualTo("127.0.0.1"); - assertThat(settings.get("http.enabled")).isEqualTo("true"); } @Test - public void enable_http_connector_different_host() throws Exception { - Props props = minProps(CLUSTER_DISABLED); - props.set(SEARCH_HTTP_PORT.getKey(), "9010"); + @UseDataProvider("clusterEnabledOrNot") + public void enable_http_connector_on_specified_port(boolean clusterEnabled) throws Exception { + String port = "" + new Random().nextInt(49151); + Props props = minProps(clusterEnabled); + props.set(SEARCH_PORT.getKey(), port); + Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + + assertThat(settings.get("http.port")).isEqualTo(port); + assertThat(settings.get("http.host")).isEqualTo("127.0.0.1"); + } + + @Test + @UseDataProvider("clusterEnabledOrNot") + public void enable_http_connector_different_host(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); props.set(SEARCH_HOST.getKey(), "127.0.0.2"); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); - assertThat(settings.get("http.port")).isEqualTo("9010"); + assertThat(settings.get("http.port")).isEqualTo("9001"); assertThat(settings.get("http.host")).isEqualTo("127.0.0.2"); - assertThat(settings.get("http.enabled")).isEqualTo("true"); } @Test - public void enable_seccomp_filter_by_default() throws Exception { - Props props = minProps(CLUSTER_DISABLED); + @UseDataProvider("clusterEnabledOrNot") + public void enable_seccomp_filter_by_default(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("bootstrap.system_call_filter")).isNull(); } @Test - public void disable_seccomp_filter_if_configured_in_search_additional_props() throws Exception { - Props props = minProps(CLUSTER_DISABLED); + @UseDataProvider("clusterEnabledOrNot") + public void disable_seccomp_filter_if_configured_in_search_additional_props(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); props.set("sonar.search.javaAdditionalOpts", "-Xmx1G -Dbootstrap.system_call_filter=false -Dfoo=bar"); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); @@ -314,8 +327,9 @@ public class EsSettingsTest { } @Test - public void disable_mmap_if_configured_in_search_additional_props() throws Exception { - Props props = minProps(CLUSTER_DISABLED); + @UseDataProvider("clusterEnabledOrNot") + public void disable_mmap_if_configured_in_search_additional_props(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); props.set("sonar.search.javaAdditionalOpts", "-Dnode.store.allow_mmap=false"); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java index 967b319e8b6..d4654af2477 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java @@ -25,9 +25,10 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.AppenderBase; import java.util.ArrayList; import java.util.List; -import org.elasticsearch.client.transport.NoNodeAvailableException; +import java.util.Optional; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.discovery.MasterNotDiscoveredException; +import org.elasticsearch.rest.RestStatus; import org.junit.Test; import org.slf4j.LoggerFactory; import org.sonar.application.es.EsConnector; @@ -41,9 +42,17 @@ import static org.mockito.Mockito.when; public class EsManagedProcessTest { @Test + public void isOperational_should_return_false_if_status_is_unknown() { + EsConnector esConnector = mock(EsConnector.class); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.empty()); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + assertThat(underTest.isOperational()).isFalse(); + } + + @Test public void isOperational_should_return_false_if_Elasticsearch_is_RED() { EsConnector esConnector = mock(EsConnector.class); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.RED); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isFalse(); } @@ -51,7 +60,7 @@ public class EsManagedProcessTest { @Test public void isOperational_should_return_true_if_Elasticsearch_is_YELLOW() { EsConnector esConnector = mock(EsConnector.class); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.YELLOW); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.YELLOW)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isTrue(); } @@ -59,7 +68,7 @@ public class EsManagedProcessTest { @Test public void isOperational_should_return_true_if_Elasticsearch_is_GREEN() { EsConnector esConnector = mock(EsConnector.class); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isTrue(); } @@ -67,11 +76,11 @@ public class EsManagedProcessTest { @Test public void isOperational_should_return_true_if_Elasticsearch_was_GREEN_once() { EsConnector esConnector = mock(EsConnector.class); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isTrue(); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.RED); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED)); assertThat(underTest.isOperational()).isTrue(); } @@ -79,8 +88,8 @@ public class EsManagedProcessTest { public void isOperational_should_retry_if_Elasticsearch_is_unreachable() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenThrow(new NoNodeAvailableException("test")) - .thenReturn(ClusterHealthStatus.GREEN); + .thenReturn(Optional.empty()) + .thenReturn(Optional.of(ClusterHealthStatus.GREEN)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isTrue(); } @@ -105,7 +114,7 @@ public class EsManagedProcessTest { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenThrow(new MasterNotDiscoveredException("Master not elected -test-")); + .thenThrow(new ElasticsearchStatusException("foobar[type=master_not_discovered_exception,acme]...", RestStatus.SERVICE_UNAVAILABLE)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isFalse(); diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java index 0c25ed6b843..3c6583b2050 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java +++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java @@ -74,7 +74,8 @@ public class ProcessProperties { SEARCH_HOST("sonar.search.host", InetAddress.getLoopbackAddress().getHostAddress()), SEARCH_PORT("sonar.search.port", "9001"), - SEARCH_HTTP_PORT("sonar.search.httpPort"), + // FIXME default is 0 until we move out of usage of TransportClient and we can put the expected default: 9002 + SEARCH_TRANSPORT_PORT("sonar.search.transportPort", "0"), SEARCH_JAVA_OPTS("sonar.search.javaOpts", "-Xmx512m -Xms512m -XX:MaxDirectMemorySize=256m -XX:+HeapDumpOnOutOfMemoryError"), SEARCH_JAVA_ADDITIONAL_OPTS("sonar.search.javaAdditionalOpts", ""), SEARCH_REPLICAS("sonar.search.replicas"), @@ -224,6 +225,8 @@ public class ProcessProperties { } fixPortIfZero(props, Property.SEARCH_HOST.getKey(), Property.SEARCH_PORT.getKey()); + // FIXME remove when transport is not used anymore in non-DCE editions: sonar.search.transportPort must not support port 0 + fixPortIfZero(props, Property.SEARCH_HOST.getKey(), Property.SEARCH_TRANSPORT_PORT.getKey()); } private Properties defaults() { diff --git a/server/sonar-server-common/build.gradle b/server/sonar-server-common/build.gradle index 5e027f81878..a6ffdd9d140 100644 --- a/server/sonar-server-common/build.gradle +++ b/server/sonar-server-common/build.gradle @@ -15,7 +15,7 @@ dependencies { compile 'com.google.guava:guava' compile 'org.slf4j:slf4j-api' compile 'com.squareup.okhttp3:okhttp' - compile 'org.elasticsearch.client:transport' + compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client' compile project(':server:sonar-db-dao') compile project(':server:sonar-db-migration') compile project(':server:sonar-process') @@ -26,6 +26,7 @@ dependencies { compileOnly 'com.google.code.findbugs:jsr305' + testCompile 'org.elasticsearch.plugin:transport-netty4-client' testCompile 'ch.qos.logback:logback-core' testCompile 'com.google.code.findbugs:jsr305' testCompile 'com.squareup.okhttp3:mockwebserver' @@ -46,6 +47,7 @@ dependencies { testFixturesApi testFixtures(project(':server:sonar-db-dao')) testFixturesCompileOnly 'com.google.code.findbugs:jsr305' + testFixturesImplementation 'org.elasticsearch.plugin:transport-netty4-client' testFixturesImplementation 'org.codelibs.elasticsearch.module:analysis-common' testFixturesImplementation 'org.elasticsearch:mocksocket' diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java index 8e01790e79e..a24c8e335bd 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java @@ -27,8 +27,9 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import javax.annotation.Nullable; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -152,9 +153,9 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe } private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) { - SearchRequestBuilder searchRequest = esClient.prepareSearch(TYPE_COMPONENT.getMainType()) - .setQuery(QueryBuilders.termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) - .setRouting(AuthorizationDoc.idOf(projectUuid)); + SearchRequest searchRequest = EsClient.prepareSearch(TYPE_COMPONENT.getMainType()) + .source(new SearchSourceBuilder().query(QueryBuilders.termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid))) + .routing(AuthorizationDoc.idOf(projectUuid)); bulkIndexer.addDeletion(searchRequest); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java index 115887d2a7e..ba0d8d3b4cb 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -31,8 +31,10 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; @@ -40,12 +42,13 @@ import org.elasticsearch.action.bulk.BulkProcessor.Listener; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -74,7 +77,8 @@ public class BulkIndexer { private static final String REFRESH_INTERVAL_SETTING = "index.refresh_interval"; private static final int DEFAULT_NUMBER_OF_SHARDS = 5; - private final EsClient client; + private final EsClient esClient; + private final IndexType indexType; private final BulkProcessor bulkProcessor; private final IndexingResult result = new IndexingResult(); @@ -86,12 +90,14 @@ public class BulkIndexer { } public BulkIndexer(EsClient client, IndexType indexType, Size size, IndexingListener indexingListener) { - this.client = client; + this.esClient = client; this.indexType = indexType; this.sizeHandler = size.createHandler(Runtime2.INSTANCE); this.indexingListener = indexingListener; BulkProcessorListener bulkProcessorListener = new BulkProcessorListener(); - this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), bulkProcessorListener) + this.bulkProcessor = BulkProcessor.builder( + client::bulkAsync, + bulkProcessorListener) .setBackoffPolicy(BackoffPolicy.exponentialBackoff()) .setBulkSize(FLUSH_BYTE_SIZE) .setBulkActions(FLUSH_ACTIONS) @@ -118,7 +124,9 @@ public class BulkIndexer { Thread.currentThread().interrupt(); throw new IllegalStateException("Elasticsearch bulk requests still being executed after 1 minute", e); } - client.prepareRefresh(indexType.getMainType().getIndex()).get(); + + esClient.refresh(indexType.getMainType().getIndex()); + sizeHandler.afterStop(this); indexingListener.onFinish(result); return result; @@ -139,50 +147,53 @@ public class BulkIndexer { bulkProcessor.add(request); } - public void addDeletion(SearchRequestBuilder searchRequest) { + public void addDeletion(SearchRequest searchRequest) { // TODO to be replaced by delete_by_query that is back in ES5 searchRequest - .addSort("_doc", SortOrder.ASC) - .setScroll(TimeValue.timeValueMinutes(5)) - .setSize(100) + .scroll(TimeValue.timeValueMinutes(5)) + .source() + .sort("_doc", SortOrder.ASC) + .size(100) // load only doc ids, not _source fields - .setFetchSource(false); + .fetchSource(false); // this search is synchronous. An optimization would be to be non-blocking, // but it requires to tracking pending requests in close(). // Same semaphore can't be reused because of potential deadlock (requires to acquire // two locks) - SearchResponse searchResponse = searchRequest.get(); + SearchResponse searchResponse = esClient.search(searchRequest); while (true) { SearchHit[] hits = searchResponse.getHits().getHits(); for (SearchHit hit : hits) { DocumentField routing = hit.field("_routing"); - DeleteRequestBuilder deleteRequestBuilder = client.prepareDelete(hit.getIndex(), hit.getType(), hit.getId()); + DeleteRequest deleteRequest = new DeleteRequest(hit.getIndex(), hit.getType(), hit.getId()); if (routing != null) { - deleteRequestBuilder.setRouting(routing.getValue()); + deleteRequest.routing(routing.getValue()); } - add(deleteRequestBuilder.request()); + add(deleteRequest); } String scrollId = searchResponse.getScrollId(); if (scrollId == null) { break; } - searchResponse = client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get(); + searchResponse = esClient.scroll(new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueMinutes(5))); if (hits.length == 0) { - client.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + esClient.clearScroll(clearScrollRequest); break; } } } public void addDeletion(IndexType indexType, String id) { - add(client.prepareDelete(indexType, id).request()); + add(new DeleteRequest(indexType.getMainType().getIndex().getName(), indexType.getMainType().getType(), id)); } public void addDeletion(IndexType indexType, String id, @Nullable String routing) { - add(client.prepareDelete(indexType, id).setRouting(routing).request()); + add(new DeleteRequest(indexType.getMainType().getIndex().getName(), indexType.getMainType().getType(), id).routing(routing)); } /** @@ -191,7 +202,7 @@ public class BulkIndexer { * <p> * Note that the parameter indexType could be removed if progress logs are not needed. */ - public static IndexingResult delete(EsClient client, IndexType indexType, SearchRequestBuilder searchRequest) { + public static IndexingResult delete(EsClient client, IndexType indexType, SearchRequest searchRequest) { BulkIndexer bulk = new BulkIndexer(client, indexType, Size.REGULAR); bulk.start(); bulk.addDeletion(searchRequest); @@ -381,13 +392,14 @@ public class BulkIndexer { .setPluralLabel("requests"); this.progress.start(); Map<String, Object> temporarySettings = new HashMap<>(); - GetSettingsResponse settingsResp = bulkIndexer.client.nativeClient().admin().indices().prepareGetSettings(index).get(); + + GetSettingsResponse settingsResp = bulkIndexer.esClient.getSettings(new GetSettingsRequest()); // deactivate replicas - int initialReplicas = Integer.parseInt(settingsResp.getSetting(index, IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); + int initialReplicas = Integer.parseInt(settingsResp.getSetting(index, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)); if (initialReplicas > 0) { - initialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas); - temporarySettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0); + initialSettings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, initialReplicas); + temporarySettings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); } // deactivate periodical refresh @@ -403,16 +415,16 @@ public class BulkIndexer { // optimize lucene segments and revert index settings // Optimization must be done before re-applying replicas: // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/ - bulkIndexer.client.prepareForceMerge(bulkIndexer.indexType.getMainType().getIndex().getName()).get(); + bulkIndexer.esClient.forcemerge(new ForceMergeRequest(bulkIndexer.indexType.getMainType().getIndex().getName())); updateSettings(bulkIndexer, initialSettings); this.progress.stop(); } private static void updateSettings(BulkIndexer bulkIndexer, Map<String, Object> settings) { - UpdateSettingsRequestBuilder req = bulkIndexer.client.nativeClient().admin().indices().prepareUpdateSettings(bulkIndexer.indexType.getMainType().getIndex().getName()); - req.setSettings(settings); - req.get(); + UpdateSettingsRequest req = new UpdateSettingsRequest(bulkIndexer.indexType.getMainType().getIndex().getName()); + req.settings(settings); + bulkIndexer.esClient.putSettings(req); } } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/package-info.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/ElasticsearchException.java index 408ecb94a59..00ffbdd61d3 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/package-info.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/ElasticsearchException.java @@ -17,7 +17,11 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -@ParametersAreNonnullByDefault -package org.sonar.server.es.request; +package org.sonar.server.es; -import javax.annotation.ParametersAreNonnullByDefault; +public class ElasticsearchException extends RuntimeException { + + public ElasticsearchException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java index 87d32d64f5a..98941a70998 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java @@ -19,159 +19,273 @@ */ package org.sonar.server.es; +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; import java.io.Closeable; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; -import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.delete.DeleteRequestBuilder; -import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.client.Client; +import java.io.IOException; +import java.util.Arrays; +import java.util.function.Supplier; +import org.apache.http.HttpHost; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.ClearScrollResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.Cancellable; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Requests; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import org.sonar.server.es.IndexType.IndexMainType; -import org.sonar.server.es.request.ProxyClearCacheRequestBuilder; -import org.sonar.server.es.request.ProxyClusterHealthRequestBuilder; -import org.sonar.server.es.request.ProxyClusterStateRequestBuilder; -import org.sonar.server.es.request.ProxyClusterStatsRequestBuilder; -import org.sonar.server.es.request.ProxyCreateIndexRequestBuilder; -import org.sonar.server.es.request.ProxyDeleteRequestBuilder; -import org.sonar.server.es.request.ProxyGetRequestBuilder; -import org.sonar.server.es.request.ProxyIndexRequestBuilder; -import org.sonar.server.es.request.ProxyIndicesExistsRequestBuilder; -import org.sonar.server.es.request.ProxyIndicesStatsRequestBuilder; -import org.sonar.server.es.request.ProxyNodesStatsRequestBuilder; -import org.sonar.server.es.request.ProxyPutMappingRequestBuilder; -import org.sonar.server.es.request.ProxyRefreshRequestBuilder; -import org.sonar.server.es.request.ProxySearchRequestBuilder; -import org.sonar.server.es.request.ProxySearchScrollRequestBuilder; - -import static java.util.Objects.requireNonNull; +import org.sonar.api.utils.log.Profiler; +import org.sonar.server.es.response.ClusterStatsResponse; +import org.sonar.server.es.response.IndicesStatsResponse; +import org.sonar.server.es.response.NodeStatsResponse; + +import static org.sonar.server.es.EsRequestDetails.computeDetailsAsString; /** - * Facade to connect to Elasticsearch node. Handles correctly errors (logging + exceptions + * Wrapper to connect to Elasticsearch node. Handles correctly errors (logging + exceptions * with context) and profiling of requests. */ public class EsClient implements Closeable { + private final RestHighLevelClient restHighLevelClient; + private final Gson gson; public static final Logger LOGGER = Loggers.get("es"); - private final Client nativeClient; + public EsClient(HttpHost... hosts) { + this(new MinimalRestHighLevelClient(hosts)); + } + + EsClient(RestHighLevelClient restHighLevelClient) { + this.restHighLevelClient = restHighLevelClient; + this.gson = new GsonBuilder().create(); + } - public EsClient(Client nativeClient) { - this.nativeClient = requireNonNull(nativeClient); + public BulkResponse bulk(BulkRequest bulkRequest) { + return execute(() -> restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT)); } - public EsClient() { - this.nativeClient = null; + public Cancellable bulkAsync(BulkRequest bulkRequest, ActionListener<BulkResponse> listener) { + return restHighLevelClient.bulkAsync(bulkRequest, RequestOptions.DEFAULT, listener); } - public RefreshRequestBuilder prepareRefresh(Index index) { - return new ProxyRefreshRequestBuilder(nativeClient()).setIndices(index.getName()); + public static SearchRequest prepareSearch(String indexName) { + return Requests.searchRequest(indexName); } - public IndicesStatsRequestBuilder prepareStats() { - return new ProxyIndicesStatsRequestBuilder(nativeClient()); + public static SearchRequest prepareSearch(IndexType.IndexMainType mainType) { + return Requests.searchRequest(mainType.getIndex().getName()).types(mainType.getType()); } - public IndicesStatsRequestBuilder prepareStats(Index index) { - return new ProxyIndicesStatsRequestBuilder(nativeClient()).setIndices(index.getName()); + public static SearchRequest prepareSearch(String index, String type) { + return Requests.searchRequest(index).types(type); } - public NodesStatsRequestBuilder prepareNodesStats(String... nodesIds) { - return new ProxyNodesStatsRequestBuilder(nativeClient()).setNodesIds(nodesIds); + public SearchResponse search(SearchRequest searchRequest) { + return execute(() -> restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(searchRequest)); } - public ClusterStatsRequestBuilder prepareClusterStats() { - return new ProxyClusterStatsRequestBuilder(nativeClient()); + public SearchResponse scroll(SearchScrollRequest searchScrollRequest) { + return execute(() -> restHighLevelClient.scroll(searchScrollRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(searchScrollRequest)); } - public ClusterStateRequestBuilder prepareState() { - return new ProxyClusterStateRequestBuilder(nativeClient()); + public ClearScrollResponse clearScroll(ClearScrollRequest clearScrollRequest) { + return execute(() -> restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT)); } - public ClusterHealthRequestBuilder prepareHealth() { - return new ProxyClusterHealthRequestBuilder(nativeClient()); + public DeleteResponse delete(DeleteRequest deleteRequest) { + return execute(() -> restHighLevelClient.delete(deleteRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(deleteRequest)); } - public void waitForStatus(ClusterHealthStatus status) { - prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForStatus(status).get(); + public RefreshResponse refresh(Index... indices) { + RefreshRequest refreshRequest = new RefreshRequest() + .indices(Arrays.stream(indices).map(Index::getName).toArray(String[]::new)); + return execute(() -> restHighLevelClient.indices().refresh(refreshRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(refreshRequest)); } - public IndicesExistsRequestBuilder prepareIndicesExist(Index index) { - return new ProxyIndicesExistsRequestBuilder(nativeClient(), index.getName()); + public ForceMergeResponse forcemerge(ForceMergeRequest forceMergeRequest) { + return execute(() -> restHighLevelClient.indices().forcemerge(forceMergeRequest, RequestOptions.DEFAULT)); } - public CreateIndexRequestBuilder prepareCreate(Index index) { - return new ProxyCreateIndexRequestBuilder(nativeClient(), index.getName()); + public AcknowledgedResponse putSettings(UpdateSettingsRequest req) { + return execute(() -> restHighLevelClient.indices().putSettings(req, RequestOptions.DEFAULT)); } - public PutMappingRequestBuilder preparePutMapping(Index index) { - return new ProxyPutMappingRequestBuilder(nativeClient()).setIndices(index.getName()); + public ClearIndicesCacheResponse clearCache(ClearIndicesCacheRequest request) { + return execute(() -> restHighLevelClient.indices().clearCache(request, RequestOptions.DEFAULT), + () -> computeDetailsAsString(request)); } - public SearchRequestBuilder prepareSearch(Index index) { - return new ProxySearchRequestBuilder(nativeClient()).setIndices(index.getName()); + public IndexResponse index(IndexRequest indexRequest) { + return execute(() -> restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(indexRequest)); } - public SearchRequestBuilder - prepareSearch(IndexMainType indexType) { - return new ProxySearchRequestBuilder(nativeClient()) - .setIndices(indexType.getIndex().getName()) - .setTypes(indexType.getType()); + public GetResponse get(GetRequest request) { + return execute(() -> restHighLevelClient.get(request, RequestOptions.DEFAULT), + () -> computeDetailsAsString(request)); } - public SearchScrollRequestBuilder prepareSearchScroll(String scrollId) { - return new ProxySearchScrollRequestBuilder(scrollId, nativeClient()); + public GetIndexResponse getIndex(GetIndexRequest getRequest) { + return execute(() -> restHighLevelClient.indices().get(getRequest, RequestOptions.DEFAULT)); } - public GetRequestBuilder prepareGet(IndexType indexType, String id) { - IndexMainType mainType = indexType.getMainType(); - return new ProxyGetRequestBuilder(nativeClient()).setIndex(mainType.getIndex().getName()).setType(mainType.getType()).setId(id); + public boolean indexExists(GetIndexRequest getIndexRequest) { + return execute(() -> restHighLevelClient.indices().exists(getIndexRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(getIndexRequest)); } - public DeleteRequestBuilder prepareDelete(IndexType indexType, String id) { - IndexMainType mainType = indexType.getMainType(); - return new ProxyDeleteRequestBuilder(nativeClient(), mainType.getIndex().getName()).setType(mainType.getType()).setId(id); + public CreateIndexResponse create(CreateIndexRequest createIndexRequest) { + return execute(() -> restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(createIndexRequest)); } - DeleteRequestBuilder prepareDelete(String index, String type, String id) { - return new ProxyDeleteRequestBuilder(nativeClient(), index).setType(type).setId(id); + public AcknowledgedResponse deleteIndex(DeleteIndexRequest deleteIndexRequest) { + return execute(() -> restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT)); } - public IndexRequestBuilder prepareIndex(IndexType indexType) { - IndexMainType mainType = indexType.getMainType(); - return new ProxyIndexRequestBuilder(nativeClient()).setIndex(mainType.getIndex().getName()).setType(mainType.getType()); + public AcknowledgedResponse putMapping(PutMappingRequest request) { + return execute(() -> restHighLevelClient.indices().putMapping(request, RequestOptions.DEFAULT), + () -> computeDetailsAsString(request)); } - public ForceMergeRequestBuilder prepareForceMerge(String indexName) { - // TODO add proxy for profiling - return nativeClient().admin().indices().prepareForceMerge(indexName) - .setMaxNumSegments(1); + public ClusterHealthResponse clusterHealth(ClusterHealthRequest clusterHealthRequest) { + return execute(() -> restHighLevelClient.cluster().health(clusterHealthRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(clusterHealthRequest)); } - public ClearIndicesCacheRequestBuilder prepareClearCache(String... indices) { - return new ProxyClearCacheRequestBuilder(nativeClient()).setIndices(indices); + public void waitForStatus(ClusterHealthStatus clusterHealthStatus) { + clusterHealth(new ClusterHealthRequest().waitForEvents(Priority.LANGUID).waitForStatus(clusterHealthStatus)); } - public Client nativeClient() { - return nativeClient; + // https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html + public NodeStatsResponse nodesStats() { + return execute(() -> { + Request request = new Request("GET", "/_nodes/stats/fs,process,jvm,indices,breaker"); + Response response = restHighLevelClient.getLowLevelClient().performRequest(request); + return NodeStatsResponse.toNodeStatsResponse(gson.fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class)); + }); + } + + // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html + public IndicesStatsResponse indicesStats(String... indices) { + return execute(() -> { + Request request = new Request("GET", "/" + (indices.length > 0 ? (String.join(",", indices) + "/") : "") + "_stats"); + request.addParameter("level", "shards"); + Response response = restHighLevelClient.getLowLevelClient().performRequest(request); + return IndicesStatsResponse.toIndicesStatsResponse(gson.fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class)); + }, () -> computeDetailsAsString(indices)); + } + + // https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html + public ClusterStatsResponse clusterStats() { + return execute(() -> { + Request request = new Request("GET", "/_cluster/stats"); + Response response = restHighLevelClient.getLowLevelClient().performRequest(request); + return ClusterStatsResponse.toClusterStatsResponse(gson.fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class)); + }); + } + + public GetSettingsResponse getSettings(GetSettingsRequest getSettingsRequest) { + return execute(() -> restHighLevelClient.indices().getSettings(getSettingsRequest, RequestOptions.DEFAULT)); + } + + public GetMappingsResponse getMapping(GetMappingsRequest getMappingsRequest) { + return execute(() -> restHighLevelClient.indices().getMapping(getMappingsRequest, RequestOptions.DEFAULT)); } @Override public void close() { - nativeClient.close(); + try { + restHighLevelClient.close(); + } catch (IOException e) { + throw new ElasticsearchException("Could not close ES Rest high level client", e); + } } + + /** + * Internal usage only + * + * @return native ES client object + */ + RestHighLevelClient nativeClient() { + return restHighLevelClient; + } + + static class MinimalRestHighLevelClient extends RestHighLevelClient { + + public MinimalRestHighLevelClient(HttpHost... hosts) { + super(RestClient.builder(hosts)); + } + + MinimalRestHighLevelClient(RestClient restClient) { + super(restClient, RestClient::close, Lists.newArrayList()); + } + } + + <R> R execute(EsRequestExecutor<R> executor) { + return execute(executor, () -> ""); + } + + <R> R execute(EsRequestExecutor<R> executor, Supplier<String> requestDetails) { + Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); + try { + return executor.execute(); + } catch (Exception e) { + throw new ElasticsearchException("Fail to execute es request" + requestDetails.get(), e); + } finally { + if (profiler.isTraceEnabled()) { + profiler.stopTrace(requestDetails.get()); + } + } + } + + @FunctionalInterface + interface EsRequestExecutor<R> { + R execute() throws IOException; + } + } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClientProvider.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClientProvider.java index 4e7a52ad257..f84189a3262 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClientProvider.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClientProvider.java @@ -20,20 +20,14 @@ package org.sonar.server.es; import com.google.common.net.HostAndPort; -import io.netty.util.ThreadDeathWatcher; -import io.netty.util.concurrent.GlobalEventExecutor; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Arrays; -import java.util.concurrent.TimeUnit; +import java.util.Collections; +import java.util.List; import java.util.stream.Collectors; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.network.NetworkModule; +import org.apache.http.HttpHost; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.join.ParentJoinPlugin; -import org.elasticsearch.percolator.PercolatorPlugin; -import org.elasticsearch.transport.Netty4Plugin; import org.picocontainer.injectors.ProviderAdapter; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.config.Configuration; @@ -42,7 +36,6 @@ import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.process.cluster.NodeType; -import static java.util.Collections.unmodifiableList; import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_TYPE; @@ -68,58 +61,41 @@ public class EsClientProvider extends ProviderAdapter { boolean clusterEnabled = config.getBoolean(CLUSTER_ENABLED.getKey()).orElse(false); boolean searchNode = !clusterEnabled || SEARCH.equals(NodeType.parse(config.get(CLUSTER_NODE_TYPE.getKey()).orElse(null))); - final TransportClient nativeClient = new MinimalTransportClient(esSettings.build()); + List<HttpHost> httpHosts; if (clusterEnabled && !searchNode) { - esSettings.put("client.transport.sniff", true); - Arrays.stream(config.getStringArray(CLUSTER_SEARCH_HOSTS.getKey())) - .map(HostAndPort::fromString) - .forEach(h -> addHostToClient(h, nativeClient)); - LOGGER.info("Connected to remote Elasticsearch: [{}]", displayedAddresses(nativeClient)); + httpHosts = getHttpHosts(config); + + LOGGER.info("Connected to remote Elasticsearch: [{}]", displayedAddresses(httpHosts)); } else { + //defaults provided in: + // * in org.sonar.process.ProcessProperties.Property.SEARCH_HOST + // * in org.sonar.process.ProcessProperties.Property.SEARCH_PORT HostAndPort host = HostAndPort.fromParts(config.get(SEARCH_HOST.getKey()).get(), config.getInt(SEARCH_PORT.getKey()).get()); - addHostToClient(host, nativeClient); - LOGGER.info("Connected to local Elasticsearch: [{}]", displayedAddresses(nativeClient)); + httpHosts = Collections.singletonList(toHttpHost(host)); + LOGGER.info("Connected to local Elasticsearch: [{}]", displayedAddresses(httpHosts)); } - cache = new EsClient(nativeClient); + cache = new EsClient(httpHosts.toArray(new HttpHost[0])); } return cache; } - private static void addHostToClient(HostAndPort host, TransportClient client) { + private static List<HttpHost> getHttpHosts(Configuration config) { + return Arrays.stream(config.getStringArray(CLUSTER_SEARCH_HOSTS.getKey())) + .map(HostAndPort::fromString) + .map(EsClientProvider::toHttpHost) + .collect(Collectors.toList()); + } + + private static HttpHost toHttpHost(HostAndPort host) { try { - client.addTransportAddress(new TransportAddress(InetAddress.getByName(host.getHost()), host.getPortOrDefault(9001))); + return new HttpHost(InetAddress.getByName(host.getHost()), host.getPortOrDefault(9001)); } catch (UnknownHostException e) { throw new IllegalStateException("Can not resolve host [" + host + "]", e); } } - private static String displayedAddresses(TransportClient nativeClient) { - return nativeClient.transportAddresses().stream().map(TransportAddress::toString).collect(Collectors.joining(", ")); - } - - static class MinimalTransportClient extends TransportClient { - - MinimalTransportClient(Settings settings) { - super(settings, unmodifiableList(Arrays.asList(Netty4Plugin.class, PercolatorPlugin.class, ParentJoinPlugin.class))); - } - - @Override - public void close() { - super.close(); - if (!NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings) - || NetworkModule.TRANSPORT_TYPE_SETTING.get(settings).equals(Netty4Plugin.NETTY_TRANSPORT_NAME)) { - try { - GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - try { - ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } + private static String displayedAddresses(List<HttpHost> httpHosts) { + return httpHosts.stream().map(HttpHost::toString).collect(Collectors.joining(", ")); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java new file mode 100644 index 00000000000..0b8e19b4083 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java @@ -0,0 +1,168 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import java.util.Arrays; +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.GetIndexRequest; + +final class EsRequestDetails { + private static final String ON_INDICES_MESSAGE = " on indices '%s'"; + private static final String ON_TYPE_MESSAGE = " on type '%s'"; + + private EsRequestDetails() { + // this is utility class only + } + + static String computeDetailsAsString(SearchRequest searchRequest) { + StringBuilder message = new StringBuilder(); + message.append(String.format("ES search request '%s'", searchRequest)); + if (searchRequest.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, Arrays.toString(searchRequest.indices()))); + } + if (searchRequest.types().length > 0) { + message.append(String.format(" on types '%s'", Arrays.toString(searchRequest.types()))); + } + return message.toString(); + } + + public static String computeDetailsAsString(SearchScrollRequest searchScrollRequest) { + return String.format("ES search scroll request for scroll id '%s'", searchScrollRequest.scroll()); + } + + static String computeDetailsAsString(DeleteRequest deleteRequest) { + return new StringBuilder() + .append("ES delete request of doc ") + .append(deleteRequest.id()) + .append(" in index ") + .append(deleteRequest.index()) + .append("/") + .append(deleteRequest.type()) + .toString(); + } + + static String computeDetailsAsString(RefreshRequest refreshRequest) { + StringBuilder message = new StringBuilder(); + message.append("ES refresh request"); + if (refreshRequest.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(refreshRequest.indices(), ","))); + } + return message.toString(); + } + + static String computeDetailsAsString(ClearIndicesCacheRequest request) { + StringBuilder message = new StringBuilder(); + message.append("ES clear cache request"); + if (request.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(request.indices(), ","))); + } + String[] fields = request.fields(); + if (fields != null && fields.length > 0) { + message.append(String.format(" on fields '%s'", StringUtils.join(fields, ","))); + } + if (request.queryCache()) { + message.append(" with filter cache"); + } + if (request.fieldDataCache()) { + message.append(" with field data cache"); + } + if (request.requestCache()) { + message.append(" with request cache"); + } + return message.toString(); + } + + static String computeDetailsAsString(IndexRequest indexRequest) { + return new StringBuilder().append("ES index request") + .append(String.format(" for key '%s'", indexRequest.id())) + .append(String.format(" on index '%s'", indexRequest.index())) + .append(String.format(ON_TYPE_MESSAGE, indexRequest.type())) + .toString(); + } + + static String computeDetailsAsString(GetRequest request) { + return new StringBuilder().append("ES get request") + .append(String.format(" for key '%s'", request.id())) + .append(String.format(" on index '%s'", request.index())) + .append(String.format(ON_TYPE_MESSAGE, request.type())) + .toString(); + } + + static String computeDetailsAsString(GetIndexRequest getIndexRequest) { + StringBuilder message = new StringBuilder(); + message.append("ES indices exists request"); + if (getIndexRequest.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(getIndexRequest.indices(), ","))); + } + return message.toString(); + } + + static String computeDetailsAsString(CreateIndexRequest createIndexRequest) { + return String.format("ES create index '%s'", createIndexRequest.index()); + } + + static String computeDetailsAsString(PutMappingRequest request) { + StringBuilder message = new StringBuilder(); + message.append("ES put mapping request"); + if (request.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(request.indices(), ","))); + } + String type = request.type(); + if (type != null) { + message.append(String.format(ON_TYPE_MESSAGE, type)); + } + String source = request.source(); + if (source != null) { + message.append(String.format(" with source '%s'", source)); + } + + return message.toString(); + } + + static String computeDetailsAsString(ClusterHealthRequest clusterHealthRequest) { + StringBuilder message = new StringBuilder(); + message.append("ES cluster health request"); + String[] indices = clusterHealthRequest.indices(); + if (indices != null && indices.length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(indices, ","))); + } + return message.toString(); + } + + static String computeDetailsAsString(String... indices) { + StringBuilder message = new StringBuilder(); + message.append("ES indices stats request"); + if (indices.length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(indices, ","))); + } + return message.toString(); + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java index cc6692ff426..e99a61b168d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java @@ -33,13 +33,13 @@ import java.util.function.Function; import java.util.regex.Pattern; import javax.annotation.CheckForNull; import javax.annotation.Nullable; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.joda.time.format.ISODateTimeFormat; import org.sonar.core.util.stream.MoreCollectors; @@ -101,8 +101,8 @@ public class EsUtils { * Optimize scolling, by specifying document sorting. * See https://www.elastic.co/guide/en/elasticsearch/reference/2.4/search-request-scroll.html#search-request-scroll */ - public static void optimizeScrollRequest(SearchRequestBuilder esSearch) { - esSearch.addSort("_doc", SortOrder.ASC); + public static void optimizeScrollRequest(SearchSourceBuilder esSearch) { + esSearch.sort("_doc", SortOrder.ASC); } /** @@ -136,9 +136,9 @@ public class EsUtils { @Override public boolean hasNext() { if (hits.isEmpty()) { - SearchScrollRequestBuilder esRequest = esClient.prepareSearchScroll(scrollId) - .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); - Collections.addAll(hits, esRequest.get().getHits().getHits()); + SearchScrollRequest esRequest = new SearchScrollRequest(scrollId) + .scroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); + Collections.addAll(hits, esClient.scroll(esRequest).getHits().getHits()); } return !hits.isEmpty(); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java index 3aa00c816da..27e7d089ae8 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java @@ -38,7 +38,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.missing.Missing; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.aggregations.metrics.Sum; import static org.sonar.api.utils.DateUtils.parseDateTime; import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_MODE_EFFORT; diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchIdResult.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchIdResult.java index 6313b59eeca..c369f0759a9 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchIdResult.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchIdResult.java @@ -36,7 +36,7 @@ public class SearchIdResult<ID> { public SearchIdResult(SearchResponse response, Function<String, ID> converter, TimeZone timeZone) { this.facets = new Facets(response, timeZone); - this.total = response.getHits().getTotalHits(); + this.total = response.getHits().getTotalHits().value; this.uuids = convertToIds(response.getHits(), converter); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchResult.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchResult.java index ddee72eb3be..996ba9afd1d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchResult.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchResult.java @@ -34,7 +34,7 @@ public class SearchResult<DOC extends BaseDoc> { public SearchResult(SearchResponse response, Function<Map<String, Object>, DOC> converter, TimeZone timeZone) { this.facets = new Facets(response, timeZone); - this.total = response.getHits().getTotalHits(); + this.total = response.getHits().getTotalHits().value; this.docs = EsUtils.convertToDocs(response.getHits(), converter); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java index a8646bad84b..41613aa45e4 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java @@ -20,8 +20,9 @@ package org.sonar.server.es.metadata; import java.util.Optional; -import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.document.DocumentField; import org.sonar.server.es.EsClient; import org.sonar.server.es.Index; @@ -90,9 +91,8 @@ public class MetadataIndexImpl implements MetadataIndex { } private Optional<String> getMetadata(String id) { - GetRequestBuilder request = esClient.prepareGet(TYPE_METADATA, id) - .setStoredFields(MetadataIndexDefinition.FIELD_VALUE); - GetResponse response = request.get(); + GetResponse response = esClient.get(new GetRequest(TYPE_METADATA.getIndex().getName(), TYPE_METADATA.getType(), id) + .storedFields(MetadataIndexDefinition.FIELD_VALUE)); if (response.isExists()) { DocumentField field = response.getField(MetadataIndexDefinition.FIELD_VALUE); return Optional.of(field.getValue()); @@ -101,10 +101,8 @@ public class MetadataIndexImpl implements MetadataIndex { } private void setMetadata(String id, String value) { - esClient.prepareIndex(TYPE_METADATA) - .setId(id) - .setSource(MetadataIndexDefinition.FIELD_VALUE, value) - .setRefreshPolicy(REFRESH_IMMEDIATE) - .get(); + esClient.index(new IndexRequest(TYPE_METADATA.getIndex().getName(), TYPE_METADATA.getType(), id) + .source(MetadataIndexDefinition.FIELD_VALUE, value) + .setRefreshPolicy(REFRESH_IMMEDIATE)); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java index bfc82efbe15..53ff8d5ec23 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java @@ -21,7 +21,7 @@ package org.sonar.server.es.newindex; import java.util.Arrays; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; public class DefaultIndexSettings { @@ -82,7 +82,7 @@ public class DefaultIndexSettings { public static Settings.Builder defaults() { Settings.Builder builder = Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put("index.refresh_interval", "30s"); Arrays.stream(DefaultIndexSettingsElement.values()) diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java index ad3517a70e7..c6c5c00af5f 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java @@ -78,7 +78,7 @@ public enum DefaultIndexSettingsElement { @Override protected void setup() { - set(TYPE, "nGram"); + set(TYPE, "ngram"); set(MIN_GRAM, MINIMUM_NGRAM_LENGTH); set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH); setList("token_chars", "letter", "digit", "punctuation", "symbol"); @@ -91,7 +91,7 @@ public enum DefaultIndexSettingsElement { @Override protected void setup() { - set(TYPE, "nGram"); + set(TYPE, "ngram"); set(MIN_GRAM, MINIMUM_NGRAM_LENGTH); set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH); setList("token_chars", "letter", "digit", "punctuation", "symbol"); @@ -101,7 +101,7 @@ public enum DefaultIndexSettingsElement { @Override protected void setup() { - set(TYPE, "edgeNGram"); + set(TYPE, "edge_ngram"); set(MIN_GRAM, MINIMUM_NGRAM_LENGTH); set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java index 74a9bf2eec8..c55fde28622 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java @@ -26,7 +26,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.stream.Stream; import javax.annotation.CheckForNull; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.sonar.api.config.Configuration; import org.sonar.server.es.Index; @@ -63,8 +63,8 @@ public abstract class NewIndex<T extends NewIndex<T>> { .orElse(settingsConfiguration.getDefaultNbOfShards()); int replicas = clusterMode ? config.getInt(SEARCH_REPLICAS.getKey()).orElse(1) : 0; - settings.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shards); - settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas); + settings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shards); + settings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas); settings.put("index.max_ngram_diff", DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH - DefaultIndexSettings.MINIMUM_NGRAM_LENGTH); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java deleted file mode 100644 index 6cb107a4eb5..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyClearCacheRequestBuilder extends ClearIndicesCacheRequestBuilder { - - public ProxyClearCacheRequestBuilder(Client client) { - super(client.admin().indices(), ClearIndicesCacheAction.INSTANCE); - } - - @Override - public ClearIndicesCacheResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public ClearIndicesCacheResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ClearIndicesCacheResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<ClearIndicesCacheResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES clear cache request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - String[] fields = request.fields(); - if (fields != null && fields.length > 0) { - message.append(String.format(" on fields '%s'", StringUtils.join(fields, ","))); - } - if (request.queryCache()) { - message.append(" with filter cache"); - } - if (request.fieldDataCache()) { - message.append(" with field data cache"); - } - if (request.requestCache()) { - message.append(" with request cache"); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java deleted file mode 100644 index d616fc0f03d..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyClusterHealthRequestBuilder extends ClusterHealthRequestBuilder { - - public ProxyClusterHealthRequestBuilder(Client client) { - super(client.admin().cluster(), ClusterHealthAction.INSTANCE); - } - - @Override - public ClusterHealthResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public ClusterHealthResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ClusterHealthResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<ClusterHealthResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES cluster health request"); - String[] indices = request.indices(); - if (indices != null && indices.length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(indices, ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java deleted file mode 100644 index 6a19906b134..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyClusterStateRequestBuilder extends ClusterStateRequestBuilder { - - public ProxyClusterStateRequestBuilder(Client client) { - super(client.admin().cluster(), ClusterStateAction.INSTANCE); - } - - @Override - public ClusterStateResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public ClusterStateResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ClusterStateResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<ClusterStateResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES cluster state request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java deleted file mode 100644 index 96f97e21bb4..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyClusterStatsRequestBuilder extends ClusterStatsRequestBuilder { - - public ProxyClusterStatsRequestBuilder(Client client) { - super(client.admin().cluster(), ClusterStatsAction.INSTANCE); - } - - @Override - public ClusterStatsResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public ClusterStatsResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ClusterStatsResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<ClusterStatsResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES cluster stats request"); - if (request.nodesIds() != null) { - message.append(String.format(" on nodes '%s'", StringUtils.join(request.nodesIds(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java deleted file mode 100644 index 6de17fc23bd..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import java.io.IOException; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyCreateIndexRequestBuilder extends CreateIndexRequestBuilder { - - private final String index; - - public ProxyCreateIndexRequestBuilder(Client client, String index) { - super(client.admin().indices(), CreateIndexAction.INSTANCE, index); - this.index = index; - } - - @Override - public CreateIndexResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public CreateIndexResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public CreateIndexResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<CreateIndexResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - public String toJson() { - try { - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject() - .field("settings") - .startObject(); - request.settings().toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject().endObject(); - builder.prettyPrint(); - return builder.toString(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public String toString() { - return String.format("ES create index '%s'", index); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java deleted file mode 100644 index 2688370d747..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.delete.DeleteAction; -import org.elasticsearch.action.delete.DeleteRequestBuilder; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyDeleteRequestBuilder extends DeleteRequestBuilder { - - public ProxyDeleteRequestBuilder(Client client, String index) { - super(client, DeleteAction.INSTANCE, index); - } - - @Override - public DeleteResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public DeleteResponse get(TimeValue timeout) { - throw new UnsupportedOperationException("Not yet implemented"); - } - - @Override - public DeleteResponse get(String timeout) { - throw new UnsupportedOperationException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<DeleteResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message - .append("ES delete request of doc ") - .append(request.id()) - .append(" in index ") - .append(request.index()) - .append("/") - .append(request.type()); - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java deleted file mode 100644 index 98c4a9b767a..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.get.GetAction; -import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyGetRequestBuilder extends GetRequestBuilder { - - public ProxyGetRequestBuilder(Client client) { - super(client, GetAction.INSTANCE); - } - - @Override - public GetResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public GetResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public GetResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<GetResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder().append("ES get request"); - message.append(String.format(" for key '%s'", request.id())); - message.append(String.format(" on index '%s'", request.index())); - message.append(String.format(" on type '%s'", request.type())); - return message.toString(); - } - -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java deleted file mode 100644 index a34165a026c..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.index.IndexAction; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyIndexRequestBuilder extends IndexRequestBuilder { - - public ProxyIndexRequestBuilder(Client client) { - super(client, IndexAction.INSTANCE); - } - - @Override - public IndexResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public IndexResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public IndexResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<IndexResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder().append("ES index request"); - message.append(String.format(" for key '%s'", request.id())); - message.append(String.format(" on index '%s'", request.index())); - message.append(String.format(" on type '%s'", request.type())); - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java deleted file mode 100644 index 14e27d25f91..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyIndicesExistsRequestBuilder extends IndicesExistsRequestBuilder { - - public ProxyIndicesExistsRequestBuilder(Client client, String... indices) { - super(client.admin().indices(), IndicesExistsAction.INSTANCE, indices); - } - - @Override - public IndicesExistsResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public IndicesExistsResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public IndicesExistsResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<IndicesExistsResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES indices exists request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java deleted file mode 100644 index 798c5724ef4..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyIndicesStatsRequestBuilder extends IndicesStatsRequestBuilder { - - public ProxyIndicesStatsRequestBuilder(Client client) { - super(client.admin().indices(), IndicesStatsAction.INSTANCE); - } - - @Override - public IndicesStatsResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public IndicesStatsResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public IndicesStatsResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<IndicesStatsResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES indices stats request"); - String[] indices = request.indices(); - if (indices != null && indices.length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(indices, ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java deleted file mode 100644 index 76d16fa02b2..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyNodesStatsRequestBuilder extends NodesStatsRequestBuilder { - - public ProxyNodesStatsRequestBuilder(Client client) { - super(client.admin().cluster(), NodesStatsAction.INSTANCE); - } - - @Override - public NodesStatsResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public NodesStatsResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public NodesStatsResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<NodesStatsResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES nodes stats request"); - if (request.nodesIds().length > 0) { - message.append(String.format(" on nodes '%s'", StringUtils.join(request.nodesIds(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java deleted file mode 100644 index 4c74b7a9de5..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyPutMappingRequestBuilder extends PutMappingRequestBuilder { - - public ProxyPutMappingRequestBuilder(Client client) { - super(client.admin().indices(), PutMappingAction.INSTANCE); - } - - @Override - public AcknowledgedResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public AcknowledgedResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public AcknowledgedResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ActionFuture<AcknowledgedResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES put mapping request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - String type = request.type(); - if (type != null) { - message.append(String.format(" on type '%s'", type)); - } - String source = request.source(); - if (source != null) { - message.append(String.format(" with source '%s'", source)); - } - - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java deleted file mode 100644 index af72e58976f..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; -import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyRefreshRequestBuilder extends RefreshRequestBuilder { - - public ProxyRefreshRequestBuilder(Client client) { - super(client.admin().indices(), RefreshAction.INSTANCE); - } - - @Override - public RefreshResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public RefreshResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public RefreshResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<RefreshResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES refresh request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java deleted file mode 100644 index 8250b2f58d3..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import java.util.Arrays; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxySearchRequestBuilder extends SearchRequestBuilder { - - public ProxySearchRequestBuilder(Client client) { - super(client, SearchAction.INSTANCE); - } - - @Override - public SearchResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public SearchResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public SearchResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<SearchResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append(String.format("ES search request '%s'", super.request)); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", Arrays.toString(request.indices()))); - } - if (request.types().length > 0) { - message.append(String.format(" on types '%s'", Arrays.toString(request.types()))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java deleted file mode 100644 index 1f56355114b..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollAction; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxySearchScrollRequestBuilder extends SearchScrollRequestBuilder { - - public ProxySearchScrollRequestBuilder(String scrollId, Client client) { - super(client, SearchScrollAction.INSTANCE, scrollId); - } - - @Override - public SearchResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public SearchResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public SearchResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<SearchResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - return String.format("ES search scroll request for scroll id '%s'", super.request().scroll()); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/ClusterStatsResponse.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/ClusterStatsResponse.java new file mode 100644 index 00000000000..a87acd45d18 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/ClusterStatsResponse.java @@ -0,0 +1,48 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; +import org.elasticsearch.cluster.health.ClusterHealthStatus; + +@Immutable +public class ClusterStatsResponse { + + private final ClusterHealthStatus healthStatus; + private final int nodeCount; + + private ClusterStatsResponse(JsonObject clusterStatsJson) { + this.healthStatus = ClusterHealthStatus.fromString(clusterStatsJson.get("status").getAsString()); + this.nodeCount = clusterStatsJson.getAsJsonObject("nodes").getAsJsonObject("count").get("total").getAsInt(); + } + + public static ClusterStatsResponse toClusterStatsResponse(JsonObject jsonObject) { + return new ClusterStatsResponse(jsonObject); + } + + public ClusterHealthStatus getHealthStatus() { + return healthStatus; + } + + public int getNodeCount() { + return nodeCount; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndexStats.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndexStats.java new file mode 100644 index 00000000000..6bbdd8ba11e --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndexStats.java @@ -0,0 +1,58 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class IndexStats { + private final String name; + private final long docCount; + private final long shardsCount; + private final long storeSizeBytes; + + private IndexStats(String name, JsonObject indexStatsJson) { + this.name = name; + this.docCount = indexStatsJson.getAsJsonObject().getAsJsonObject("primaries").getAsJsonObject("docs").get("count").getAsLong(); + this.shardsCount = indexStatsJson.getAsJsonObject().getAsJsonObject("shards").size(); + this.storeSizeBytes = indexStatsJson.getAsJsonObject().getAsJsonObject("primaries").getAsJsonObject("store").get("size_in_bytes").getAsLong(); + } + + static IndexStats toIndexStats(String name, JsonObject indexStatsJson) { + return new IndexStats(name, indexStatsJson); + } + + public String getName() { + return name; + } + + public long getDocCount() { + return docCount; + } + + public long getShardsCount() { + return shardsCount; + } + + public long getStoreSizeBytes() { + return storeSizeBytes; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStats.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStats.java new file mode 100644 index 00000000000..75928920bfd --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStats.java @@ -0,0 +1,65 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class IndicesStats { + private static final String MEMORY_SIZE_IN_BYTES_ATTRIBUTE_NAME = "memory_size_in_bytes"; + private final long storeSizeInBytes; + private final long translogSizeInBytes; + private final long requestCacheMemorySizeInBytes; + private final long fieldDataMemorySizeInBytes; + private final long queryCacheMemorySizeInBytes; + + private IndicesStats(JsonObject indicesStatsJson) { + this.storeSizeInBytes = indicesStatsJson.getAsJsonObject("store").get("size_in_bytes").getAsLong(); + this.translogSizeInBytes = indicesStatsJson.getAsJsonObject("translog").get("size_in_bytes").getAsLong(); + this.requestCacheMemorySizeInBytes = indicesStatsJson.getAsJsonObject("request_cache").get(MEMORY_SIZE_IN_BYTES_ATTRIBUTE_NAME).getAsLong(); + this.fieldDataMemorySizeInBytes = indicesStatsJson.getAsJsonObject("fielddata").get(MEMORY_SIZE_IN_BYTES_ATTRIBUTE_NAME).getAsLong(); + this.queryCacheMemorySizeInBytes = indicesStatsJson.getAsJsonObject("query_cache").get(MEMORY_SIZE_IN_BYTES_ATTRIBUTE_NAME).getAsLong(); + } + + public static IndicesStats toIndicesStats(JsonObject jsonObject) { + return new IndicesStats(jsonObject); + } + + public long getStoreSizeInBytes() { + return storeSizeInBytes; + } + + public long getTranslogSizeInBytes() { + return translogSizeInBytes; + } + + public long getRequestCacheMemorySizeInBytes() { + return requestCacheMemorySizeInBytes; + } + + public long getFieldDataMemorySizeInBytes() { + return fieldDataMemorySizeInBytes; + } + + public long getQueryCacheMemorySizeInBytes() { + return queryCacheMemorySizeInBytes; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStatsResponse.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStatsResponse.java new file mode 100644 index 00000000000..0762c395024 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStatsResponse.java @@ -0,0 +1,50 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.util.Collection; +import java.util.Map; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class IndicesStatsResponse { + + private final ImmutableMap<String, IndexStats> indexStatsMap; + + private IndicesStatsResponse(JsonObject indicesStats) { + ImmutableMap.Builder<String, IndexStats> builder = ImmutableMap.builder(); + for (Map.Entry<String, JsonElement> indexStats : indicesStats.getAsJsonObject("indices").entrySet()) { + builder.put(indexStats.getKey(), IndexStats.toIndexStats(indexStats.getKey(), indexStats.getValue().getAsJsonObject())); + } + this.indexStatsMap = builder.build(); + } + + public static IndicesStatsResponse toIndicesStatsResponse(JsonObject jsonObject) { + return new IndicesStatsResponse(jsonObject); + } + + public Collection<IndexStats> getAllIndexStats() { + return indexStatsMap.values(); + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/JvmStats.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/JvmStats.java new file mode 100644 index 00000000000..31369d8afb0 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/JvmStats.java @@ -0,0 +1,64 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class JvmStats { + private final long heapUsedPercent; + private final long heapUsedInBytes; + private final long heapMaxInBytes; + private final long nonHeapUsedInBytes; + private final long threadCount; + + private JvmStats(JsonObject jvmStatsJson) { + this.heapUsedPercent = jvmStatsJson.getAsJsonObject("mem").get("heap_used_percent").getAsLong(); + this.heapUsedInBytes = jvmStatsJson.getAsJsonObject("mem").get("heap_used_in_bytes").getAsLong(); + this.heapMaxInBytes = jvmStatsJson.getAsJsonObject("mem").get("heap_max_in_bytes").getAsLong(); + this.nonHeapUsedInBytes = jvmStatsJson.getAsJsonObject("mem").get("non_heap_used_in_bytes").getAsLong(); + this.threadCount = jvmStatsJson.getAsJsonObject("threads").get("count").getAsLong(); + } + + public static JvmStats toJvmStats(JsonObject jvmStatsJson) { + return new JvmStats(jvmStatsJson); + } + + public long getHeapUsedPercent() { + return heapUsedPercent; + } + + public long getHeapUsedInBytes() { + return heapUsedInBytes; + } + + public long getHeapMaxInBytes() { + return heapMaxInBytes; + } + + public long getNonHeapUsedInBytes() { + return nonHeapUsedInBytes; + } + + public long getThreadCount() { + return threadCount; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java new file mode 100644 index 00000000000..f7249380e43 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java @@ -0,0 +1,115 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class NodeStats { + private static final String PROCESS_ATTRIBUTE_NAME = "process"; + private static final String BREAKERS_ATTRIBUTE_NAME = "breakers"; + + private final String name; + private final String host; + + private final long cpuUsage; + private final long openFileDescriptors; + private final long maxFileDescriptors; + private final long diskAvailableBytes; + + private final long fieldDataCircuitBreakerLimit; + private final long fieldDataCircuitBreakerEstimation; + private final long requestCircuitBreakerLimit; + private final long requestCircuitBreakerEstimation; + + private final JvmStats jvmStats; + private final IndicesStats indicesStats; + + private NodeStats(JsonObject nodeStatsJson) { + this.name = nodeStatsJson.get("name").getAsString(); + this.host = nodeStatsJson.get("host").getAsString(); + + this.cpuUsage = nodeStatsJson.getAsJsonObject(PROCESS_ATTRIBUTE_NAME).getAsJsonObject("cpu").get("percent").getAsLong(); + this.openFileDescriptors = nodeStatsJson.getAsJsonObject(PROCESS_ATTRIBUTE_NAME).get("open_file_descriptors").getAsLong(); + this.maxFileDescriptors = nodeStatsJson.getAsJsonObject(PROCESS_ATTRIBUTE_NAME).get("max_file_descriptors").getAsLong(); + this.diskAvailableBytes = nodeStatsJson.getAsJsonObject("fs").getAsJsonObject("total").get("available_in_bytes").getAsLong(); + + this.fieldDataCircuitBreakerLimit = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("fielddata").get("limit_size_in_bytes").getAsLong(); + this.fieldDataCircuitBreakerEstimation = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("fielddata").get("estimated_size_in_bytes").getAsLong(); + this.requestCircuitBreakerLimit = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("request").get("limit_size_in_bytes").getAsLong(); + this.requestCircuitBreakerEstimation = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("request").get("estimated_size_in_bytes").getAsLong(); + + this.jvmStats = JvmStats.toJvmStats(nodeStatsJson.getAsJsonObject("jvm")); + this.indicesStats = IndicesStats.toIndicesStats(nodeStatsJson.getAsJsonObject("indices")); + } + + public static NodeStats toNodeStats(JsonObject jsonObject) { + return new NodeStats(jsonObject); + } + + public String getName() { + return name; + } + + public String getHost() { + return host; + } + + public long getCpuUsage() { + return cpuUsage; + } + + public long getOpenFileDescriptors() { + return openFileDescriptors; + } + + public long getMaxFileDescriptors() { + return maxFileDescriptors; + } + + public long getDiskAvailableBytes() { + return diskAvailableBytes; + } + + public long getFieldDataCircuitBreakerLimit() { + return fieldDataCircuitBreakerLimit; + } + + public long getFieldDataCircuitBreakerEstimation() { + return fieldDataCircuitBreakerEstimation; + } + + public long getRequestCircuitBreakerLimit() { + return requestCircuitBreakerLimit; + } + + public long getRequestCircuitBreakerEstimation() { + return requestCircuitBreakerEstimation; + } + + public JvmStats getJvmStats() { + return jvmStats; + } + + public IndicesStats getIndicesStats() { + return indicesStats; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStatsResponse.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStatsResponse.java new file mode 100644 index 00000000000..774459487a2 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStatsResponse.java @@ -0,0 +1,51 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.common.collect.ImmutableList; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.util.List; +import java.util.Map; +import javax.annotation.concurrent.Immutable; +import org.sonar.core.util.stream.MoreCollectors; + +@Immutable +public class NodeStatsResponse { + + private final ImmutableList<NodeStats> nodeStats; + + private NodeStatsResponse(JsonObject nodeStatsJson) { + this.nodeStats = nodeStatsJson.getAsJsonObject("nodes").entrySet().stream() + .map(Map.Entry::getValue) + .map(JsonElement::getAsJsonObject) + .map(NodeStats::toNodeStats) + .collect(MoreCollectors.toList()); + } + + public static NodeStatsResponse toNodeStatsResponse(JsonObject jsonObject) { + return new NodeStatsResponse(jsonObject); + } + + public List<NodeStats> getNodeStats() { + return this.nodeStats; + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 3e78f12b19f..69233bf3c1c 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -28,7 +28,8 @@ import java.util.Iterator; import java.util.List; import java.util.Set; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sonar.api.resources.Qualifiers; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -262,22 +263,22 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { } private IndexRequest newIndexRequest(IssueDoc issue) { - return esClient.prepareIndex(TYPE_ISSUE.getMainType()) - .setId(issue.getId()) - .setRouting(issue.getRouting().orElseThrow(() -> new IllegalStateException("IssueDoc should define a routing"))) - .setSource(issue.getFields()) - .request(); + return new IndexRequest(TYPE_ISSUE.getMainType().getIndex().getName(), TYPE_ISSUE.getMainType().getType()) + .id(issue.getId()) + .routing(issue.getRouting().orElseThrow(() -> new IllegalStateException("IssueDoc should define a routing"))) + .source(issue.getFields()); } private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) { - SearchRequestBuilder search = esClient.prepareSearch(TYPE_ISSUE.getMainType()) - .setRouting(AuthorizationDoc.idOf(projectUuid)) - .setQuery(boolQuery().must(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid))); + SearchRequest search = EsClient.prepareSearch(TYPE_ISSUE.getMainType()) + .routing(AuthorizationDoc.idOf(projectUuid)) + .source(new SearchSourceBuilder().query(boolQuery().must(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid)))); + bulkIndexer.addDeletion(search); } private static EsQueueDto createQueueDto(String docId, String docIdType, String projectUuid) { - return EsQueueDto.create(TYPE_ISSUE.format(), docId, docIdType, projectUuid); + return EsQueueDto.create(TYPE_ISSUE.format(), docId, docIdType, AuthorizationDoc.idOf(projectUuid)); } private BulkIndexer createBulkIndexer(Size size, IndexingListener listener) { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java index 92387e6b754..02f9f2a1c92 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java @@ -27,8 +27,9 @@ import java.util.Map; import java.util.Set; import javax.annotation.Nullable; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.db.DbClient; @@ -184,8 +185,8 @@ public class ActiveRuleIndexer implements ResilientIndexer { RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid); if (profile == null) { // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule - SearchRequestBuilder search = esClient.prepareSearch(TYPE_ACTIVE_RULE.getMainType()) - .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid))); + SearchRequest search = EsClient.prepareSearch(TYPE_ACTIVE_RULE.getMainType()) + .source(new SearchSourceBuilder().query(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid)))); profileResult = BulkIndexer.delete(esClient, TYPE_ACTIVE_RULE, search); } else { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java index e30b69aed3c..ead195c2e16 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -31,7 +31,7 @@ import java.util.function.Function; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -46,6 +46,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -147,38 +148,40 @@ public class RuleIndex { } public SearchIdResult<String> search(RuleQuery query, SearchOptions options) { - SearchRequestBuilder esSearch = client.prepareSearch(TYPE_RULE); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); QueryBuilder qb = buildQuery(query); Map<String, QueryBuilder> filters = buildFilters(query); if (!options.getFacets().isEmpty()) { for (AggregationBuilder aggregation : getFacets(query, options, qb, filters).values()) { - esSearch.addAggregation(aggregation); + sourceBuilder.aggregation(aggregation); } } - setSorting(query, esSearch); - setPagination(options, esSearch); + setSorting(query, sourceBuilder); + setPagination(options, sourceBuilder); BoolQueryBuilder fb = boolQuery(); for (QueryBuilder filterBuilder : filters.values()) { fb.must(filterBuilder); } - esSearch.setQuery(boolQuery().must(qb).filter(fb)); - return new SearchIdResult<>(esSearch.get(), input -> input, system2.getDefaultTimeZone()); + sourceBuilder.query(boolQuery().must(qb).filter(fb)); + + SearchRequest esSearch = EsClient.prepareSearch(TYPE_RULE) + .source(sourceBuilder); + + return new SearchIdResult<>(client.search(esSearch), input -> input, system2.getDefaultTimeZone()); } /** * Return all rule uuids matching the search query, without pagination nor facets */ public Iterator<String> searchAll(RuleQuery query) { - SearchRequestBuilder esSearch = client - .prepareSearch(TYPE_RULE) - .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - optimizeScrollRequest(esSearch); + optimizeScrollRequest(sourceBuilder); QueryBuilder qb = buildQuery(query); Map<String, QueryBuilder> filters = buildFilters(query); @@ -187,8 +190,13 @@ public class RuleIndex { fb.must(filterBuilder); } - esSearch.setQuery(boolQuery().must(qb).filter(fb)); - SearchResponse response = esSearch.get(); + sourceBuilder.query(boolQuery().must(qb).filter(fb)); + + SearchRequest esSearch = EsClient.prepareSearch(TYPE_RULE) + .scroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) + .source(sourceBuilder); + + SearchResponse response = client.search(esSearch); return scrollIds(client, response, i -> i); } @@ -561,7 +569,7 @@ public class RuleIndex { return new StickyFacetBuilder(query, filters, null, BucketOrder.compound(BucketOrder.count(false), BucketOrder.key(true))); } - private static void setSorting(RuleQuery query, SearchRequestBuilder esSearch) { + private static void setSorting(RuleQuery query, SearchSourceBuilder esSearch) { /* integrate Query Sort */ String queryText = query.getQueryText(); if (query.getSortField() != null) { @@ -571,13 +579,13 @@ public class RuleIndex { } else { sort.order(SortOrder.DESC); } - esSearch.addSort(sort); + esSearch.sort(sort); } else if (StringUtils.isNotEmpty(queryText)) { - esSearch.addSort(SortBuilders.scoreSort()); + esSearch.sort(SortBuilders.scoreSort()); } else { - esSearch.addSort(appendSortSuffixIfNeeded(FIELD_RULE_UPDATED_AT), SortOrder.DESC); + esSearch.sort(appendSortSuffixIfNeeded(FIELD_RULE_UPDATED_AT), SortOrder.DESC); // deterministic sort when exactly the same updated_at (same millisecond) - esSearch.addSort(appendSortSuffixIfNeeded(FIELD_RULE_KEY), SortOrder.ASC); + esSearch.sort(appendSortSuffixIfNeeded(FIELD_RULE_KEY), SortOrder.ASC); } } @@ -588,9 +596,9 @@ public class RuleIndex { : ""); } - private static void setPagination(SearchOptions options, SearchRequestBuilder esSearch) { - esSearch.setFrom(options.getOffset()); - esSearch.setSize(options.getLimit()); + private static void setPagination(SearchOptions options, SearchSourceBuilder esSearch) { + esSearch.from(options.getOffset()); + esSearch.size(options.getLimit()); } public List<String> listTags(@Nullable String query, int size) { @@ -611,13 +619,13 @@ public class RuleIndex { .map(s -> new IncludeExclude(s, null)) .ifPresent(termsAggregation::includeExclude); - SearchRequestBuilder request = client - .prepareSearch(TYPE_RULE.getMainType()) - .setQuery(matchAllQuery()) - .setSize(0) - .addAggregation(termsAggregation); + SearchRequest request = EsClient.prepareSearch(TYPE_RULE.getMainType()) + .source(new SearchSourceBuilder() + .query(matchAllQuery()) + .size(0) + .aggregation(termsAggregation)); - SearchResponse esResponse = request.get(); + SearchResponse esResponse = client.search(request); return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME_FOR_TAGS)); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java index 5dbd1803e0a..75d142cd6ee 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java @@ -23,12 +23,14 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; @@ -69,15 +71,17 @@ public class UserIndex { public List<UserDoc> getAtMostThreeActiveUsersForScmAccount(String scmAccount) { List<UserDoc> result = new ArrayList<>(); if (!StringUtils.isEmpty(scmAccount)) { - SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.TYPE_USER) - .setQuery(boolQuery().must(matchAllQuery()).filter( - boolQuery() - .must(termQuery(FIELD_ACTIVE, true)) - .should(termQuery(FIELD_LOGIN, scmAccount)) - .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_EMAIL), scmAccount)) - .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_SCM_ACCOUNTS), scmAccount)))) - .setSize(3); - for (SearchHit hit : request.get().getHits().getHits()) { + SearchResponse response = esClient.search(EsClient.prepareSearch(UserIndexDefinition.TYPE_USER) + .source(new SearchSourceBuilder() + .query(boolQuery().must(matchAllQuery()).filter( + boolQuery() + .must(termQuery(FIELD_ACTIVE, true)) + .should(termQuery(FIELD_LOGIN, scmAccount)) + .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_EMAIL), scmAccount)) + .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_SCM_ACCOUNTS), scmAccount)) + .minimumShouldMatch(1))) + .size(3))); + for (SearchHit hit : response.getHits().getHits()) { result.add(new UserDoc(hit.getSourceAsMap())); } } @@ -85,10 +89,10 @@ public class UserIndex { } public SearchResult<UserDoc> search(UserQuery userQuery, SearchOptions options) { - SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.TYPE_USER) - .setSize(options.getLimit()) - .setFrom(options.getOffset()) - .addSort(FIELD_NAME, SortOrder.ASC); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .size(options.getLimit()) + .from(options.getOffset()) + .sort(FIELD_NAME, SortOrder.ASC); BoolQueryBuilder filter = boolQuery().must(termQuery(FIELD_ACTIVE, true)); userQuery.getOrganizationUuid() @@ -109,9 +113,9 @@ public class UserIndex { .operator(Operator.AND); } - request.setQuery(boolQuery().must(esQuery).filter(filter)); - - return new SearchResult<>(request.get(), UserDoc::new, system2.getDefaultTimeZone()); + SearchRequest request = EsClient.prepareSearch(UserIndexDefinition.TYPE_USER) + .source(searchSourceBuilder.query(boolQuery().must(esQuery).filter(filter))); + return new SearchResult<>(esClient.search(request), UserDoc::new, system2.getDefaultTimeZone()); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java index 83eff7fe0a9..3ed04ff5f49 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java @@ -20,10 +20,13 @@ package org.sonar.server.view.index; import java.util.List; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; @@ -45,14 +48,15 @@ public class ViewIndex { } public List<String> findAllViewUuids() { - SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.TYPE_VIEW) - .addSort("_doc", SortOrder.ASC) - .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) - .setFetchSource(false) - .setSize(100) - .setQuery(matchAllQuery()); + SearchRequest esSearch = EsClient.prepareSearch(ViewIndexDefinition.TYPE_VIEW) + .source(new SearchSourceBuilder() + .sort("_doc", SortOrder.ASC) + .fetchSource(false) + .size(100) + .query(matchAllQuery())) + .scroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); - SearchResponse response = esSearch.get(); + SearchResponse response = esClient.search(esSearch); List<String> result = newArrayList(); while (true) { List<SearchHit> hits = newArrayList(response.getHits()); @@ -60,12 +64,12 @@ public class ViewIndex { result.add(hit.getId()); } String scrollId = response.getScrollId(); - response = esClient.prepareSearchScroll(scrollId) - .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) - .get(); + response = esClient.scroll(new SearchScrollRequest().scrollId(scrollId).scroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES))); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { - esClient.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + esClient.clearScroll(clearScrollRequest); break; } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java index c1a5be8d107..10b119153ac 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.index.IndexRequest; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; @@ -128,9 +129,7 @@ public class ViewIndexer implements ResilientIndexer { private void clearLookupCache(String viewUuid) { try { - esClient.prepareClearCache() - .setQueryCache(true) - .get(); + esClient.clearCache(new ClearIndicesCacheRequest().queryCache(true)); } catch (Exception e) { throw new IllegalStateException(String.format("Unable to clear lookup cache of view '%s'", viewUuid), e); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java index 1589043bd0b..af3d714d8ce 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java @@ -22,6 +22,7 @@ package org.sonar.server.component.index; import java.util.Arrays; import java.util.Collection; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.System2; @@ -32,6 +33,7 @@ import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentUpdateDto; import org.sonar.db.es.EsQueueDto; import org.sonar.db.organization.OrganizationDto; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.es.IndexingResult; import org.sonar.server.es.ProjectIndexer; @@ -257,9 +259,9 @@ public class ComponentIndexerTest { private void assertThatComponentHasName(ComponentDto component, String expectedName) { SearchHit[] hits = es.client() - .prepareSearch(TYPE_COMPONENT.getMainType()) - .setQuery(matchQuery(SORTABLE_ANALYZER.subField(FIELD_NAME), expectedName)) - .get() + .search(EsClient.prepareSearch(TYPE_COMPONENT.getMainType()) + .source(new SearchSourceBuilder() + .query(matchQuery(SORTABLE_ANALYZER.subField(FIELD_NAME), expectedName)))) .getHits() .getHits(); assertThat(hits) diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java index edaeb253d73..52a0ef0a73b 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -20,13 +20,17 @@ package org.sonar.server.es; import com.google.common.collect.ImmutableMap; +import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.sonar.api.impl.utils.TestSystem2; @@ -112,8 +116,8 @@ public class BulkIndexerTest { es.putDocuments(TYPE_FAKE, docs); assertThat(count()).isEqualTo(max); - SearchRequestBuilder req = es.client().prepareSearch(TYPE_FAKE) - .setQuery(QueryBuilders.rangeQuery(FakeIndexDefinition.INT_FIELD).gte(removeFrom)); + SearchRequest req = EsClient.prepareSearch(TYPE_FAKE) + .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery(FakeIndexDefinition.INT_FIELD).gte(removeFrom))); BulkIndexer.delete(es.client(), TYPE_FAKE, req); assertThat(count()).isEqualTo(removeFrom); @@ -197,9 +201,13 @@ public class BulkIndexerTest { } private int replicas() { - GetSettingsResponse settingsResp = es.client().nativeClient().admin().indices() - .prepareGetSettings(INDEX).get(); - return Integer.parseInt(settingsResp.getSetting(INDEX, IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); + try { + GetSettingsResponse settingsResp = es.client().nativeClient().indices() + .getSettings(new GetSettingsRequest().indices(INDEX), RequestOptions.DEFAULT); + return Integer.parseInt(settingsResp.getSetting(INDEX, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)); + } catch (IOException e) { + throw new IllegalStateException("Could not get index settings", e); + } } private IndexRequest newIndexRequest(int intField) { diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientProviderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientProviderTest.java index 7a8b5edbb7e..3ec60eb6237 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientProviderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientProviderTest.java @@ -21,8 +21,8 @@ package org.sonar.server.es; import java.net.InetAddress; import org.assertj.core.api.Condition; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.RestHighLevelClient; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -39,6 +39,7 @@ import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_TYPE; import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_HOSTS; import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST; import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT; +import static org.sonar.process.ProcessProperties.Property.SEARCH_TRANSPORT_PORT; public class EsClientProviderTest { @@ -50,29 +51,31 @@ public class EsClientProviderTest { private MapSettings settings = new MapSettings(); private EsClientProvider underTest = new EsClientProvider(); - private String localhost; + private String localhostHostname; @Before public void setUp() throws Exception { // mandatory property settings.setProperty(CLUSTER_NAME.getKey(), "the_cluster_name"); - localhost = InetAddress.getLocalHost().getHostAddress(); + localhostHostname = InetAddress.getLocalHost().getHostName(); } @Test public void connection_to_local_es_when_cluster_mode_is_disabled() { settings.setProperty(CLUSTER_ENABLED.getKey(), false); - settings.setProperty(SEARCH_HOST.getKey(), localhost); - settings.setProperty(SEARCH_PORT.getKey(), 8080); + settings.setProperty(SEARCH_HOST.getKey(), localhostHostname); + settings.setProperty(SEARCH_PORT.getKey(), 9000); + settings.setProperty(SEARCH_TRANSPORT_PORT.getKey(), 8080); EsClient client = underTest.provide(settings.asConfig()); - TransportClient transportClient = (TransportClient) client.nativeClient(); - assertThat(transportClient.transportAddresses()).hasSize(1); - TransportAddress address = transportClient.transportAddresses().get(0); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(8080); - assertThat(logTester.logs(LoggerLevel.INFO)).has(new Condition<>(s -> s.contains("Connected to local Elasticsearch: [" + localhost + ":8080]"), "")); + RestHighLevelClient nativeClient = client.nativeClient(); + assertThat(nativeClient.getLowLevelClient().getNodes()).hasSize(1); + Node node = nativeClient.getLowLevelClient().getNodes().get(0); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(9000); + + assertThat(logTester.logs(LoggerLevel.INFO)).has(new Condition<>(s -> s.contains("Connected to local Elasticsearch: [http://" + localhostHostname + ":9000]"), "")); // keep in cache assertThat(underTest.provide(settings.asConfig())).isSameAs(client); @@ -82,37 +85,41 @@ public class EsClientProviderTest { public void connection_to_remote_es_nodes_when_cluster_mode_is_enabled_and_local_es_is_disabled() { settings.setProperty(CLUSTER_ENABLED.getKey(), true); settings.setProperty(CLUSTER_NODE_TYPE.getKey(), "application"); - settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s:8080,%s:8081", localhost, localhost)); + settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s:8080,%s:8081", localhostHostname, localhostHostname)); EsClient client = underTest.provide(settings.asConfig()); - TransportClient transportClient = (TransportClient) client.nativeClient(); - assertThat(transportClient.transportAddresses()).hasSize(2); - TransportAddress address = transportClient.transportAddresses().get(0); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(8080); - address = transportClient.transportAddresses().get(1); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(8081); - assertThat(logTester.logs(LoggerLevel.INFO)).has(new Condition<>(s -> s.contains("Connected to remote Elasticsearch: [" + localhost + ":8080, " + localhost + ":8081]"), "")); + RestHighLevelClient nativeClient = client.nativeClient(); + assertThat(nativeClient.getLowLevelClient().getNodes()).hasSize(2); + + Node node = nativeClient.getLowLevelClient().getNodes().get(0); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(8080); + + node = nativeClient.getLowLevelClient().getNodes().get(1); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(8081); + + assertThat(logTester.logs(LoggerLevel.INFO)) + .has(new Condition<>(s -> s.contains("Connected to remote Elasticsearch: [http://" + localhostHostname + ":8080, http://" + localhostHostname + ":8081]"), "")); // keep in cache assertThat(underTest.provide(settings.asConfig())).isSameAs(client); } @Test - public void es_client_provider_must_throw_ISE_when_incorrect_port_is_used_when_search_disabled() { + public void es_client_provider_must_throw_IAE_when_incorrect_port_is_used_when_search_disabled() { settings.setProperty(CLUSTER_ENABLED.getKey(), true); settings.setProperty(CLUSTER_NODE_TYPE.getKey(), "application"); - settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s:100000,%s:8081", localhost, localhost)); + settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s:100000,%s:8081", localhostHostname, localhostHostname)); expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage(format("Port number out of range: %s:100000", localhost)); + expectedException.expectMessage(format("Port number out of range: %s:100000", localhostHostname)); underTest.provide(settings.asConfig()); } @Test - public void es_client_provider_must_throw_ISE_when_incorrect_port_is_used() { + public void es_client_provider_must_throw_IAE_when_incorrect_port_is_used() { settings.setProperty(CLUSTER_ENABLED.getKey(), true); settings.setProperty(CLUSTER_NODE_TYPE.getKey(), "search"); settings.setProperty(SEARCH_HOST.getKey(), "localhost"); @@ -128,18 +135,22 @@ public class EsClientProviderTest { public void es_client_provider_must_add_default_port_when_not_specified() { settings.setProperty(CLUSTER_ENABLED.getKey(), true); settings.setProperty(CLUSTER_NODE_TYPE.getKey(), "application"); - settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s,%s:8081", localhost, localhost)); + settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s,%s:8081", localhostHostname, localhostHostname)); EsClient client = underTest.provide(settings.asConfig()); - TransportClient transportClient = (TransportClient) client.nativeClient(); - assertThat(transportClient.transportAddresses()).hasSize(2); - TransportAddress address = transportClient.transportAddresses().get(0); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(9001); - address = transportClient.transportAddresses().get(1); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(8081); - assertThat(logTester.logs(LoggerLevel.INFO)).has(new Condition<>(s -> s.contains("Connected to remote Elasticsearch: [" + localhost + ":9001, " + localhost + ":8081]"), "")); + RestHighLevelClient nativeClient = client.nativeClient(); + assertThat(nativeClient.getLowLevelClient().getNodes()).hasSize(2); + + Node node = nativeClient.getLowLevelClient().getNodes().get(0); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(9001); + + node = nativeClient.getLowLevelClient().getNodes().get(1); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(8081); + + assertThat(logTester.logs(LoggerLevel.INFO)) + .has(new Condition<>(s -> s.contains("Connected to remote Elasticsearch: [http://" + localhostHostname + ":9001, http://" + localhostHostname + ":8081]"), "")); // keep in cache assertThat(underTest.provide(settings.asConfig())).isSameAs(client); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientStopperTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientStopperTest.java index 664637caf7b..4c786cb5e64 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientStopperTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientStopperTest.java @@ -23,8 +23,7 @@ import org.junit.Test; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; - +import static org.mockito.Mockito.verifyNoMoreInteractions; public class EsClientStopperTest { @@ -34,7 +33,7 @@ public class EsClientStopperTest { @Test public void stop_client() { underTest.start(); - verifyZeroInteractions(client); + verifyNoMoreInteractions(client); underTest.stop(); verify(client).close(); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java index e485491a86d..f08e0d3b403 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java @@ -19,51 +19,233 @@ */ package org.sonar.server.es; -import org.junit.Rule; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Objects; +import org.apache.http.HttpEntity; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; import org.junit.Test; -import org.sonar.server.es.newindex.FakeIndexDefinition; -import org.sonar.server.es.request.ProxyClusterHealthRequestBuilder; -import org.sonar.server.es.request.ProxyClusterStateRequestBuilder; -import org.sonar.server.es.request.ProxyClusterStatsRequestBuilder; -import org.sonar.server.es.request.ProxyCreateIndexRequestBuilder; -import org.sonar.server.es.request.ProxyDeleteRequestBuilder; -import org.sonar.server.es.request.ProxyGetRequestBuilder; -import org.sonar.server.es.request.ProxyIndicesExistsRequestBuilder; -import org.sonar.server.es.request.ProxyIndicesStatsRequestBuilder; -import org.sonar.server.es.request.ProxyNodesStatsRequestBuilder; -import org.sonar.server.es.request.ProxyPutMappingRequestBuilder; -import org.sonar.server.es.request.ProxyRefreshRequestBuilder; -import org.sonar.server.es.request.ProxySearchRequestBuilder; -import org.sonar.server.es.request.ProxySearchScrollRequestBuilder; +import org.mockito.ArgumentMatcher; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class EsClientTest { + private static final String EXAMPLE_CLUSTER_STATS_JSON = "{" + + " \"status\": \"yellow\"," + + " \"nodes\": {" + + " \"count\": {" + + " \"total\": 3" + + " }" + + " }" + + "}"; - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); + private static final String EXAMPLE_INDICES_STATS_JSON = "{" + + " \"indices\": {" + + " \"index-1\": {" + + " \"primaries\": {" + + " \"docs\": {" + + " \"count\": 1234" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 56789" + + " }" + + " }," + + " \"shards\": {" + + " \"shard-1\": {}," + + " \"shard-2\": {}" + + " }" + + " }," + + " \"index-2\": {" + + " \"primaries\": {" + + " \"docs\": {" + + " \"count\": 42" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 123" + + " }" + + " }," + + " \"shards\": {" + + " \"shard-1\": {}," + + " \"shard-2\": {}" + + " }" + + " }" + + " }" + + "}"; + + private final static String EXAMPLE_NODE_STATS_JSON = "{" + + " \"nodes\": {" + + " \"YnKPZcbGRamRQGxjErLWoQ\": {" + + " \"name\": \"sonarqube\"," + + " \"host\": \"127.0.0.1\"," + + " \"indices\": {" + + " \"docs\": {" + + " \"count\": 13557" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 8670970" + + " }," + + " \"query_cache\": {" + + " \"memory_size_in_bytes\": 0" + + " }," + + " \"fielddata\": {" + + " \"memory_size_in_bytes\": 4880" + + " }," + + " \"translog\": {" + + " \"size_in_bytes\": 8274137" + + " }," + + " \"request_cache\": {" + + " \"memory_size_in_bytes\": 0" + + " }" + + " }," + + " \"process\": {" + + " \"open_file_descriptors\": 296," + + " \"max_file_descriptors\": 10240," + + " \"cpu\": {" + + " \"percent\": 7" + + " }" + + " }," + + " \"jvm\": {" + + " \"mem\": {" + + " \"heap_used_in_bytes\": 158487160," + + " \"heap_used_percent\": 30," + + " \"heap_max_in_bytes\": 518979584," + + " \"non_heap_used_in_bytes\": 109066592" + + " }," + + " \"threads\": {" + + " \"count\": 70" + + " }" + + " }," + + " \"fs\": {" + + " \"total\": {" + + " \"total_in_bytes\": 250685575168," + + " \"free_in_bytes\": 142843138048," + + " \"available_in_bytes\": 136144027648" + + " }" + + " }," + + " \"breakers\": {" + + " \"request\": {" + + " \"limit_size_in_bytes\": 311387750," + + " \"estimated_size_in_bytes\": 0" + + " }," + + " \"fielddata\": {" + + " \"limit_size_in_bytes\": 207591833," + + " \"estimated_size_in_bytes\": 4880" + + " }" + + " }" + + " }" + + " }" + + "}"; + + RestClient restClient = mock(RestClient.class); + RestHighLevelClient client = new EsClient.MinimalRestHighLevelClient(restClient); + + EsClient underTest = new EsClient(client); @Test - public void proxify_requests() { - Index fakesIndex = Index.simple("fakes"); - IndexType.IndexMainType fakeMainType = IndexType.main(fakesIndex, "fake"); - - EsClient underTest = es.client(); - assertThat(underTest.nativeClient()).isNotNull(); - assertThat(underTest.prepareClusterStats()).isInstanceOf(ProxyClusterStatsRequestBuilder.class); - assertThat(underTest.prepareCreate(fakesIndex)).isInstanceOf(ProxyCreateIndexRequestBuilder.class); - assertThat(underTest.prepareDelete(fakeMainType, "my_id")).isInstanceOf(ProxyDeleteRequestBuilder.class); - assertThat(underTest.prepareIndicesExist(fakesIndex)).isInstanceOf(ProxyIndicesExistsRequestBuilder.class); - assertThat(underTest.prepareGet(fakeMainType, "1")).isInstanceOf(ProxyGetRequestBuilder.class); - assertThat(underTest.prepareHealth()).isInstanceOf(ProxyClusterHealthRequestBuilder.class); - assertThat(underTest.prepareNodesStats()).isInstanceOf(ProxyNodesStatsRequestBuilder.class); - assertThat(underTest.preparePutMapping(fakesIndex)).isInstanceOf(ProxyPutMappingRequestBuilder.class); - assertThat(underTest.prepareRefresh(fakesIndex)).isInstanceOf(ProxyRefreshRequestBuilder.class); - assertThat(underTest.prepareSearch(fakesIndex)).isInstanceOf(ProxySearchRequestBuilder.class); - assertThat(underTest.prepareSearchScroll("1234")).isInstanceOf(ProxySearchScrollRequestBuilder.class); - assertThat(underTest.prepareState()).isInstanceOf(ProxyClusterStateRequestBuilder.class); - assertThat(underTest.prepareStats(fakesIndex)).isInstanceOf(ProxyIndicesStatsRequestBuilder.class); + public void should_close_client() throws IOException { + underTest.close(); + verify(restClient).close(); + } + @Test(expected = ElasticsearchException.class) + public void should_rethrow_ex_when_close_client_throws() throws IOException { + doThrow(IOException.class).when(restClient).close(); underTest.close(); } + + @Test + public void should_call_node_stats_api() throws Exception { + HttpEntity entity = mock(HttpEntity.class); + when(entity.getContent()).thenReturn(new ByteArrayInputStream(EXAMPLE_NODE_STATS_JSON.getBytes())); + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(entity); + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_nodes/stats/fs,process,jvm,indices,breaker")))) + .thenReturn(response); + + assertThat(underTest.nodesStats()).isNotNull(); + } + + @Test(expected = ElasticsearchException.class) + public void should_rethrow_ex_on_node_stat_fail() throws Exception { + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_nodes/stats/fs,process,jvm,indices,breaker")))) + .thenThrow(IOException.class); + underTest.nodesStats(); + } + + @Test + public void should_call_indices_stat_api() throws Exception { + HttpEntity entity = mock(HttpEntity.class); + when(entity.getContent()).thenReturn(new ByteArrayInputStream(EXAMPLE_INDICES_STATS_JSON.getBytes())); + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(entity); + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_stats")))) + .thenReturn(response); + + assertThat(underTest.indicesStats()).isNotNull(); + } + + @Test(expected = ElasticsearchException.class) + public void should_rethrow_ex_on_indices_stat_fail() throws Exception { + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_stats")))) + .thenThrow(IOException.class); + underTest.indicesStats(); + } + + @Test + public void should_call_cluster_stat_api() throws Exception { + HttpEntity entity = mock(HttpEntity.class); + when(entity.getContent()).thenReturn(new ByteArrayInputStream(EXAMPLE_CLUSTER_STATS_JSON.getBytes())); + + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(entity); + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_cluster/stats")))) + .thenReturn(response); + + assertThat(underTest.clusterStats()).isNotNull(); + } + + @Test(expected = ElasticsearchException.class) + public void should_rethrow_ex_on_cluster_stat_fail() throws Exception { + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_cluster/stats")))) + .thenThrow(IOException.class); + underTest.clusterStats(); + } + + static class RawRequestMatcher implements ArgumentMatcher<Request> { + String endpoint; + String method; + + RawRequestMatcher(String method, String endpoint) { + Objects.requireNonNull(endpoint); + Objects.requireNonNull(method); + this.endpoint = endpoint; + this.method = method; + } + + @Override + public boolean matches(Request request) { + return endpoint.equals(request.getEndpoint()) && method.equals(request.getMethod()); + } + } + } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java new file mode 100644 index 00000000000..fe2436cea8e --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java @@ -0,0 +1,154 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.client.Requests; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.common.unit.TimeValue; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EsRequestDetailsTest { + + @Test + public void should_format_SearchRequest() { + SearchRequest searchRequest = Requests.searchRequest("index") + .types("type"); + assertThat(EsRequestDetails.computeDetailsAsString(searchRequest)) + .isEqualTo("ES search request 'SearchRequest{searchType=QUERY_THEN_FETCH, indices=[index]," + + " indicesOptions=IndicesOptions[ignore_unavailable=false, allow_no_indices=true," + + " expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false," + + " allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false," + + " ignore_throttled=true], types=[type], routing='null', preference='null', requestCache=null," + + " scroll=null, maxConcurrentShardRequests=0, batchedReduceSize=512, preFilterShardSize=null," + + " allowPartialSearchResults=null, localClusterAlias=null, getOrCreateAbsoluteStartMillis=-1," + + " ccsMinimizeRoundtrips=true, source={}}' on indices '[index]' on types '[type]'"); + } + + @Test + public void should_format_search_SearchScrollRequest() { + SearchScrollRequest scrollRequest = Requests.searchScrollRequest("scroll-id") + .scroll(TimeValue.ZERO); + assertThat(EsRequestDetails.computeDetailsAsString(scrollRequest)) + .isEqualTo("ES search scroll request for scroll id 'Scroll{keepAlive=0s}'"); + } + + @Test + public void should_format_DeleteRequest() { + DeleteRequest deleteRequest = new DeleteRequest() + .index("some-index") + .type("some-type") + .id("some-id"); + assertThat(EsRequestDetails.computeDetailsAsString(deleteRequest)) + .isEqualTo("ES delete request of doc some-id in index some-index/some-type"); + } + + @Test + public void should_format_RefreshRequest() { + RefreshRequest deleteRequest = new RefreshRequest() + .indices("index-1", "index-2"); + assertThat(EsRequestDetails.computeDetailsAsString(deleteRequest)) + .isEqualTo("ES refresh request on indices 'index-1,index-2'"); + } + + @Test + public void should_format_ClearIndicesCacheRequest() { + ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest() + .indices("index-1") + .fields("field-1") + .queryCache(true) + .fieldDataCache(true) + .requestCache(true); + assertThat(EsRequestDetails.computeDetailsAsString(clearIndicesCacheRequest)) + .isEqualTo("ES clear cache request on indices 'index-1' on fields 'field-1' with filter cache with field data cache with request cache"); + } + + @Test + public void should_format_IndexRequest() { + IndexRequest indexRequest = new IndexRequest() + .index("index-1") + .type("type-1") + .id("id-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(indexRequest)) + .isEqualTo("ES index request for key 'id-1' on index 'index-1' on type 'type-1'"); + } + + @Test + public void should_format_GetRequest() { + GetRequest request = new GetRequest() + .index("index-1") + .type("type-1") + .id("id-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES get request for key 'id-1' on index 'index-1' on type 'type-1'"); + } + + @Test + public void should_format_GetIndexRequest() { + GetIndexRequest request = new GetIndexRequest("index-1", "index-2"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES indices exists request on indices 'index-1,index-2'"); + } + + @Test + public void should_format_CreateIndexRequest() { + CreateIndexRequest request = new CreateIndexRequest("index-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES create index 'index-1'"); + } + + @Test + public void should_format_PutMappingRequest() { + PutMappingRequest request = new PutMappingRequest("index-1") + .type("type-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES put mapping request on indices 'index-1' on type 'type-1'"); + } + + @Test + public void should_format_ClusterHealthRequest() { + ClusterHealthRequest request = new ClusterHealthRequest("index-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES cluster health request on indices 'index-1'"); + } + + @Test + public void should_format_IndicesStats() { + assertThat(EsRequestDetails.computeDetailsAsString("index-1", "index-2")) + .isEqualTo("ES indices stats request on indices 'index-1,index-2'"); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsUtilsTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsUtilsTest.java index 3b883bdc172..9a068062d83 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsUtilsTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsUtilsTest.java @@ -21,6 +21,7 @@ package org.sonar.server.es; import java.util.Date; import java.util.List; +import org.apache.lucene.search.TotalHits; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.junit.Test; @@ -34,14 +35,14 @@ public class EsUtilsTest { @Test public void convertToDocs_empty() { - SearchHits hits = new SearchHits(new SearchHit[] {}, 0, 0); + SearchHits hits = new SearchHits(new SearchHit[] {}, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0); List<BaseDoc> docs = EsUtils.convertToDocs(hits, IssueDoc::new); assertThat(docs).isEmpty(); } @Test public void convertToDocs() { - SearchHits hits = new SearchHits(new SearchHit[] {new SearchHit(16)}, 1, 1); + SearchHits hits = new SearchHits(new SearchHit[] {new SearchHit(16)}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1); List<BaseDoc> docs = EsUtils.convertToDocs(hits, IssueDoc::new); assertThat(docs).hasSize(1); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java index df7601cf480..2a81194dd77 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java @@ -24,7 +24,7 @@ import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.util.Map; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.junit.Rule; import org.junit.Test; @@ -229,8 +229,8 @@ public class NewIndexTest { public void default_shards_and_replicas(Index index) { NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -239,8 +239,8 @@ public class NewIndexTest { settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); } @Test @@ -249,9 +249,9 @@ public class NewIndexTest { settings.setProperty("sonar.search." + index.getName() + ".shards", "3"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSetting(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("3"); + assertThat(newIndex.getSetting(IndexMetadata.SETTING_NUMBER_OF_SHARDS)).isEqualTo("3"); // keep default value - assertThat(newIndex.getSetting(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSetting(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -259,7 +259,7 @@ public class NewIndexTest { public void default_number_of_replicas_on_standalone_instance_must_be_0(Index index) { NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -268,7 +268,7 @@ public class NewIndexTest { settings.setProperty(CLUSTER_ENABLED.getKey(), "false"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -277,7 +277,7 @@ public class NewIndexTest { settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); } @Test @@ -287,7 +287,7 @@ public class NewIndexTest { settings.setProperty(SEARCH_REPLICAS.getKey(), "0"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -297,7 +297,7 @@ public class NewIndexTest { settings.setProperty(SEARCH_REPLICAS.getKey(), "3"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("3"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("3"); } @Test diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java deleted file mode 100644 index e3974c80d44..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyClearCacheRequestBuilderTest { - - @Rule - public EsTester es = EsTester.create(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void clear_cache() { - ClearIndicesCacheRequestBuilder requestBuilder = es.client().prepareClearCache(); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareClearCache().toString()).isEqualTo("ES clear cache request"); - assertThat(es.client().prepareClearCache("rules").toString()).isEqualTo("ES clear cache request on indices 'rules'"); - assertThat(es.client().prepareClearCache().setFields("key").toString()).isEqualTo("ES clear cache request on fields 'key'"); - assertThat(es.client().prepareClearCache().setFieldDataCache(true).toString()).isEqualTo("ES clear cache request with field data cache"); - assertThat(es.client().prepareClearCache().setRequestCache(true).toString()).isEqualTo("ES clear cache request with request cache"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - ClearIndicesCacheRequestBuilder requestBuilder = es.client().prepareClearCache(); - requestBuilder.get(); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void no_trace_logs() { - logTester.setLevel(LoggerLevel.DEBUG); - ClearIndicesCacheRequestBuilder requestBuilder = es.client().prepareClearCache(); - requestBuilder.get(); - - assertThat(logTester.logs()).isEmpty(); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareClearCache().get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareClearCache().get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareClearCache().execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java deleted file mode 100644 index 4d051019190..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import java.util.Locale; -import java.util.Random; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; - -import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyCreateIndexRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void create_index() { - CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndex()); - requestBuilder.get(); - } - - @Test - public void to_string() { - Index index = generateNewIndex(); - assertThat(es.client().prepareCreate(index).toString()).contains("ES create index '" + index.getName() + "'"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndex()); - requestBuilder.get(); - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareCreate(generateNewIndex()).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareCreate(generateNewIndex()).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareCreate(generateNewIndex()).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - - private static Index generateNewIndex(){ - String name = randomAlphabetic(10).toLowerCase(Locale.ENGLISH); - return new Random().nextBoolean() ? Index.simple(name) : Index.withRelations(name); - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java deleted file mode 100644 index f0b5da674a9..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyDeleteRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - @Rule - public LogTester logTester = new LogTester(); - - private final Index index = Index.simple("fakes"); - private final IndexType.IndexMainType mainType = IndexType.main(index, "fake"); - - @Test - public void delete() { - es.client().prepareDelete(mainType, "the_id").get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareDelete(mainType, "the_id").toString()).isEqualTo("ES delete request of doc the_id in index fakes/fake"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - es.client().prepareDelete(mainType, "the_id").get(); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareDelete(mainType, "the_id").get("1"); - fail(); - } catch (UnsupportedOperationException e) { - assertThat(e).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareDelete(mainType, "the_id").get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (UnsupportedOperationException e) { - assertThat(e).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareDelete(mainType, "the_id").execute(); - fail(); - } catch (UnsupportedOperationException e) { - assertThat(e).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java deleted file mode 100644 index 9c92ee0d1af..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import com.tngtech.java.junit.dataprovider.DataProvider; -import com.tngtech.java.junit.dataprovider.DataProviderRunner; -import com.tngtech.java.junit.dataprovider.UseDataProvider; -import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; -import org.sonar.server.es.IndexType; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; -import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; - -@RunWith(DataProviderRunner.class) -public class ProxyGetRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareGet(TYPE_FAKE, "ruleKey") - .get(); - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - @UseDataProvider("mainAndRelationWithUnknownIndex") - public void prepareGet_fails_if_index_unknown(IndexType indexType) { - GetRequestBuilder requestBuilder = es.client().prepareGet(indexType, "rule1"); - try { - requestBuilder.get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES get request for key 'rule1' on index 'unknown' on type 'test'"); - } - } - - @DataProvider - public static Object[][] mainAndRelationWithUnknownIndex() { - IndexType.IndexMainType mainType = IndexType.main(Index.withRelations("unknown"), "test"); - return new Object[][] { - {mainType}, - {IndexType.relation(mainType, "donut")} - }; - } - - @Test - public void get_with_string_timeout_is_not_implemented() { - try { - es.client().prepareGet(TYPE_FAKE, "ruleKey").get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareGet(TYPE_FAKE, "ruleKey").get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareGet(TYPE_FAKE, "ruleKey").execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java deleted file mode 100644 index 1e91d8c551f..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import com.tngtech.java.junit.dataprovider.DataProvider; -import com.tngtech.java.junit.dataprovider.DataProviderRunner; -import com.tngtech.java.junit.dataprovider.UseDataProvider; -import org.elasticsearch.action.DocWriteResponse.Result; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.IndexType.IndexMainType; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; -import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; - -@RunWith(DataProviderRunner.class) -public class ProxyIndexRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void index_with_index_type_and_id() { - IndexResponse response = es.client().prepareIndex(TYPE_FAKE) - .setSource(FakeIndexDefinition.newDoc(42).getFields()) - .get(); - assertThat(response.getResult()).isSameAs(Result.CREATED); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - IndexResponse response = es.client().prepareIndex(TYPE_FAKE) - .setSource(FakeIndexDefinition.newDoc(42).getFields()) - .get(); - assertThat(response.getResult()).isSameAs(Result.CREATED); - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - @UseDataProvider("mainOrRelationType") - public void fail_if_bad_query(IndexType indexType) { - IndexRequestBuilder requestBuilder = es.client().prepareIndex(indexType); - try { - requestBuilder.get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'foo' on type 'bar'"); - } - } - - @DataProvider - public static Object[][] mainOrRelationType() { - IndexMainType mainType = IndexType.main(Index.withRelations("foo"), "bar"); - return new Object[][] { - {mainType}, - {IndexType.relation(mainType, "donut")} - }; - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareIndex(TYPE_FAKE).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareIndex(TYPE_FAKE).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void do_not_support_execute_method() { - try { - es.client().prepareIndex(TYPE_FAKE).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java deleted file mode 100644 index 41d9ac3fe78..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyIndicesExistsRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void exists() { - assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get().isExists()).isTrue(); - assertThat(es.client().prepareIndicesExist(Index.simple("unknown")).get().isExists()).isFalse(); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get(); - - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void to_string() { - assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).toString()).isEqualTo("ES indices exists request on indices 'fakes'"); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java deleted file mode 100644 index a366b7889a5..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyIndicesStatsRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void stats() { - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).setIndices("rules").toString()).isEqualTo("ES indices stats request on indices 'rules'"); - assertThat(es.client().prepareStats().toString()).isEqualTo("ES indices stats request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(); - - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void fail_to_stats() { - try { - es.client().prepareStats(Index.simple("unknown")).get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES indices stats request on indices 'unknown'"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java deleted file mode 100644 index 1c964603d52..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ProxyNodesStatsRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Test - public void stats() { - es.client().prepareNodesStats().get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareNodesStats().setNodesIds("node1").toString()).isEqualTo("ES nodes stats request on nodes 'node1'"); - assertThat(es.client().prepareNodesStats().toString()).isEqualTo("ES nodes stats request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareNodesStats().get(); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - thrown.expect(IllegalStateException.class); - thrown.expectMessage("Not yet implemented"); - - es.client().prepareNodesStats(FakeIndexDefinition.INDEX).get("1"); - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - thrown.expect(IllegalStateException.class); - thrown.expectMessage("Not yet implemented"); - - es.client().prepareNodesStats(FakeIndexDefinition.INDEX).get(TimeValue.timeValueMinutes(1)); - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - thrown.expect(UnsupportedOperationException.class); - thrown.expectMessage("execute() should not be called as it's used for asynchronous"); - - es.client().prepareNodesStats(FakeIndexDefinition.INDEX).execute(); - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java deleted file mode 100644 index fe5f47700c2..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import com.google.common.collect.ImmutableMap; -import java.util.HashMap; -import java.util.Map; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyPutMappingRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void put_mapping() { - PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR) - .setType(FakeIndexDefinition.TYPE) - .setSource(mapDomain()); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).setSource(mapDomain()).toString()) - .isEqualTo("ES put mapping request on indices 'fakes' with source '{\"dynamic\":false,\"_all\":{\"enabled\":false}}'"); - assertThat(es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).setType(FakeIndexDefinition.TYPE).setSource(mapDomain()).toString()) - .isEqualTo("ES put mapping request on indices 'fakes' on type 'fake' with source '{\"dynamic\":false,\"_all\":{\"enabled\":false}}'"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR) - .setType(FakeIndexDefinition.TYPE) - .setSource(mapDomain()); - requestBuilder.get(); - - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void fail_on_bad_query() { - try { - es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES put mapping request"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - - protected static Map mapDomain() { - Map<String, Object> mapping = new HashMap<>(); - mapping.put("dynamic", false); - mapping.put("_all", ImmutableMap.of("enabled", false)); - return mapping; - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java deleted file mode 100644 index 47da09b91cc..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyRefreshRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void refresh() { - RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).toString()).isEqualTo("ES refresh request on indices 'fakes'"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR); - requestBuilder.get(); - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void fail_to_refresh() { - try { - es.client().prepareRefresh(Index.simple("unknown")).get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES refresh request on indices 'unknown'"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java deleted file mode 100644 index 8a73f114897..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxySearchRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void search() { - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).setTypes(FakeIndexDefinition.TYPE).toString()).contains("ES search request '") - .contains("' on indices '[fakes]' on types '[fake]'"); - assertThat(es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).toString()) - .contains("ES search request '") - .contains("' on indices '[fakes]'"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(); - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void fail_to_search_bad_query() { - try { - es.client().prepareSearch(Index.simple("unknown")).get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()) - .contains("Fail to execute ES search request 'SearchRequest{") - .contains("}' on indices '[unknown]'"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java deleted file mode 100644 index fe7308f942e..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxySearchScrollRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR) - .setScroll(TimeValue.timeValueMinutes(1)) - .get(); - logTester.clear(); - es.client().prepareSearchScroll(response.getScrollId()).get(); - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void no_trace_logs() { - logTester.setLevel(LoggerLevel.DEBUG); - - SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR) - .setScroll(TimeValue.timeValueMinutes(1)) - .get(); - logTester.clear(); - es.client().prepareSearchScroll(response.getScrollId()).get(); - assertThat(logTester.logs()).isEmpty(); - } - - @Test - public void fail_to_search_bad_query() { - try { - es.client().prepareSearchScroll("unknown").get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES search scroll request for scroll id 'null'"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareSearchScroll("scrollId").get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareSearchScroll("scrollId").get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareSearchScroll("scrollId").execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java deleted file mode 100644 index d3b164fcbd7..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyWebServerHealthRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void state() { - ClusterHealthRequestBuilder requestBuilder = es.client().prepareHealth(); - ClusterHealthResponse state = requestBuilder.get(); - assertThat(state.getStatus()).isEqualTo(ClusterHealthStatus.GREEN); - } - - @Test - public void to_string() { - assertThat(es.client().prepareHealth().toString()).isEqualTo("ES cluster health request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - ClusterHealthRequestBuilder requestBuilder = es.client().prepareHealth(); - ClusterHealthResponse state = requestBuilder.get(); - assertThat(state.getStatus()).isEqualTo(ClusterHealthStatus.GREEN); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareHealth().get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareHealth().get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareHealth().execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStateRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStateRequestBuilderTest.java deleted file mode 100644 index a48918ba7c7..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStateRequestBuilderTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyWebServerStateRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void state() { - ClusterStateRequestBuilder requestBuilder = es.client().prepareState(); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareState().setIndices("rules").toString()).isEqualTo("ES cluster state request on indices 'rules'"); - assertThat(es.client().prepareState().toString()).isEqualTo("ES cluster state request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - ClusterStateRequestBuilder requestBuilder = es.client().prepareState(); - requestBuilder.get(); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareState().get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareState().get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareState().execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStatsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStatsRequestBuilderTest.java deleted file mode 100644 index 83173c978bd..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStatsRequestBuilderTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyWebServerStatsRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void stats() { - ClusterStatsRequestBuilder requestBuilder = es.client().prepareClusterStats(); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareClusterStats().setNodesIds("node1").toString()).isEqualTo("ES cluster stats request on nodes 'node1'"); - assertThat(es.client().prepareClusterStats().toString()).isEqualTo("ES cluster stats request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - ClusterStatsRequestBuilder requestBuilder = es.client().prepareClusterStats(); - requestBuilder.get(); - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareClusterStats().get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareClusterStats().get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareClusterStats().execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/response/ClusterStatsResponseTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/ClusterStatsResponseTest.java new file mode 100644 index 00000000000..2682789a603 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/ClusterStatsResponseTest.java @@ -0,0 +1,52 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ClusterStatsResponseTest { + private static final String EXAMPLE_JSON = "{" + + " \"status\": \"yellow\"," + + " \"nodes\": {" + + " \"count\": {" + + " \"total\": 3" + + " }" + + " }" + + "}"; + + @Test + public void should_parse_example_json() { + JsonObject jsonObject = getExampleAsJsonObject(); + ClusterStatsResponse clusterStatsResponse = ClusterStatsResponse.toClusterStatsResponse(jsonObject); + + assertThat(clusterStatsResponse.getHealthStatus()).isEqualTo(ClusterHealthStatus.YELLOW); + assertThat(clusterStatsResponse.getNodeCount()).isEqualTo(3); + } + + private static JsonObject getExampleAsJsonObject() { + return new Gson().fromJson(EXAMPLE_JSON, JsonObject.class); + } + +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/response/IndicesStatsResponseTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/IndicesStatsResponseTest.java new file mode 100644 index 00000000000..0ed3ac313c5 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/IndicesStatsResponseTest.java @@ -0,0 +1,89 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import java.util.Collection; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class IndicesStatsResponseTest { + private static final String EXAMPLE_JSON = "{" + + " \"indices\": {" + + " \"index-1\": {" + + " \"primaries\": {" + + " \"docs\": {" + + " \"count\": 1234" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 56789" + + " }" + + " }," + + " \"shards\": {" + + " \"shard-1\": {}," + + " \"shard-2\": {}" + + " }" + + " }," + + " \"index-2\": {" + + " \"primaries\": {" + + " \"docs\": {" + + " \"count\": 42" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 123" + + " }" + + " }," + + " \"shards\": {" + + " \"shard-1\": {}," + + " \"shard-2\": {}" + + " }" + + " }" + + " }" + + "}"; + + @Test + public void should_parse_example_json() { + JsonObject jsonObject = getExampleAsJsonObject(); + IndicesStatsResponse indicesStatsResponse = IndicesStatsResponse.toIndicesStatsResponse(jsonObject); + + Collection<IndexStats> allIndexStats = indicesStatsResponse.getAllIndexStats(); + assertThat(allIndexStats) + .hasSize(2) + .extracting("name") + .contains("index-1", "index-2"); + + IndexStats indexStats = allIndexStats.stream().filter(i -> i.getName().equals("index-1")).findFirst().get(); + assertThat(indexStats.getDocCount()).isEqualTo(1234); + assertThat(indexStats.getShardsCount()).isEqualTo(2); + assertThat(indexStats.getStoreSizeBytes()).isEqualTo(56789); + + indexStats = allIndexStats.stream().filter(i -> i.getName().equals("index-2")).findFirst().get(); + assertThat(indexStats.getDocCount()).isEqualTo(42); + assertThat(indexStats.getStoreSizeBytes()).isEqualTo(123); + assertThat(indexStats.getShardsCount()).isEqualTo(2); + } + + private static JsonObject getExampleAsJsonObject() { + return new Gson().fromJson(EXAMPLE_JSON, JsonObject.class); + } + +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java new file mode 100644 index 00000000000..6c7c6e7eef3 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java @@ -0,0 +1,135 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class NodeStatsResponseTest { + private final static String EXAMPLE_JSON = "{" + + " \"nodes\": {" + + " \"YnKPZcbGRamRQGxjErLWoQ\": {" + + " \"name\": \"sonarqube\"," + + " \"host\": \"127.0.0.1\"," + + " \"indices\": {" + + " \"docs\": {" + + " \"count\": 13557" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 8670970" + + " }," + + " \"query_cache\": {" + + " \"memory_size_in_bytes\": 0" + + " }," + + " \"fielddata\": {" + + " \"memory_size_in_bytes\": 4880" + + " }," + + " \"translog\": {" + + " \"size_in_bytes\": 8274137" + + " }," + + " \"request_cache\": {" + + " \"memory_size_in_bytes\": 0" + + " }" + + " }," + + " \"process\": {" + + " \"open_file_descriptors\": 296," + + " \"max_file_descriptors\": 10240," + + " \"cpu\": {" + + " \"percent\": 7" + + " }" + + " }," + + " \"jvm\": {" + + " \"mem\": {" + + " \"heap_used_in_bytes\": 158487160," + + " \"heap_used_percent\": 30," + + " \"heap_max_in_bytes\": 518979584," + + " \"non_heap_used_in_bytes\": 109066592" + + " }," + + " \"threads\": {" + + " \"count\": 70" + + " }" + + " }," + + " \"fs\": {" + + " \"total\": {" + + " \"total_in_bytes\": 250685575168," + + " \"free_in_bytes\": 142843138048," + + " \"available_in_bytes\": 136144027648" + + " }" + + " }," + + " \"breakers\": {" + + " \"request\": {" + + " \"limit_size_in_bytes\": 311387750," + + " \"estimated_size_in_bytes\": 1" + + " }," + + " \"fielddata\": {" + + " \"limit_size_in_bytes\": 207591833," + + " \"estimated_size_in_bytes\": 4880" + + " }" + + " }" + + " }" + + " }" + + "}"; + + @Test + public void should_parse_example_json() { + JsonObject jsonObject = getExampleAsJsonObject(); + NodeStatsResponse nodeStatsResponse = NodeStatsResponse.toNodeStatsResponse(jsonObject); + + assertThat(nodeStatsResponse.getNodeStats()).hasSize(1); + + NodeStats nodeStats = nodeStatsResponse.getNodeStats().get(0); + assertThat(nodeStats.getName()).isEqualTo("sonarqube"); + assertThat(nodeStats.getHost()).isEqualTo("127.0.0.1"); + assertThat(nodeStats.getCpuUsage()).isEqualTo(7); + + assertThat(nodeStats.getOpenFileDescriptors()).isEqualTo(296); + assertThat(nodeStats.getMaxFileDescriptors()).isEqualTo(10240); + assertThat(nodeStats.getDiskAvailableBytes()).isEqualTo(136144027648L); + + assertThat(nodeStats.getFieldDataCircuitBreakerLimit()).isEqualTo(207591833); + assertThat(nodeStats.getFieldDataCircuitBreakerEstimation()).isEqualTo(4880); + assertThat(nodeStats.getRequestCircuitBreakerLimit()).isEqualTo(311387750L); + assertThat(nodeStats.getRequestCircuitBreakerEstimation()).isEqualTo(1); + + JvmStats jvmStats = nodeStats.getJvmStats(); + assertThat(jvmStats).isNotNull(); + assertThat(jvmStats.getHeapUsedPercent()).isEqualTo(30); + assertThat(jvmStats.getHeapUsedInBytes()).isEqualTo(158487160); + assertThat(jvmStats.getHeapMaxInBytes()).isEqualTo(518979584); + assertThat(jvmStats.getNonHeapUsedInBytes()).isEqualTo(109066592); + assertThat(jvmStats.getThreadCount()).isEqualTo(70); + + IndicesStats indicesStats = nodeStats.getIndicesStats(); + assertThat(indicesStats).isNotNull(); + assertThat(indicesStats.getStoreSizeInBytes()).isEqualTo(8670970); + assertThat(indicesStats.getTranslogSizeInBytes()).isEqualTo(8274137); + assertThat(indicesStats.getRequestCacheMemorySizeInBytes()).isEqualTo(0); + assertThat(indicesStats.getFieldDataMemorySizeInBytes()).isEqualTo(4880); + assertThat(indicesStats.getQueryCacheMemorySizeInBytes()).isEqualTo(0); + + } + + private static JsonObject getExampleAsJsonObject() { + return new Gson().fromJson(EXAMPLE_JSON, JsonObject.class); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java index 1d615d974f0..dbb76a0d77d 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java @@ -29,7 +29,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; import org.junit.Test; import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java index da52873dbf5..d054d06760d 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java @@ -23,8 +23,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.function.Consumer; import java.util.function.Predicate; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.sonar.api.resources.Qualifiers; @@ -50,6 +51,7 @@ import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; +import static org.sonar.server.es.EsClient.prepareSearch; import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_CREATION; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_DELETION; @@ -288,12 +290,13 @@ public class ProjectMeasuresIndexerTest { } private void assertThatProjectHasTag(ComponentDto project, String expectedTag) { - SearchRequestBuilder request = es.client() - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setQuery(boolQuery() - .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())) - .filter(termQuery(FIELD_TAGS, expectedTag))); - assertThat(request.get().getHits().getHits()) + SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(new SearchSourceBuilder() + .query(boolQuery() + .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())) + .filter(termQuery(FIELD_TAGS, expectedTag)))); + + assertThat(es.client().search(request).getHits().getHits()) .extracting(SearchHit::getId) .contains(project.uuid()); } @@ -323,13 +326,15 @@ public class ProjectMeasuresIndexerTest { } private void assertThatQualifierIs(String qualifier, String... componentsUuid) { - SearchRequestBuilder request = es.client() - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setQuery(boolQuery() + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .query(boolQuery() .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())) .filter(termQuery(FIELD_QUALIFIER, qualifier)) .filter(termsQuery(FIELD_UUID, componentsUuid))); - assertThat(request.get().getHits().getHits()) + + SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(searchSourceBuilder); + assertThat(es.client().search(request).getHits().getHits()) .extracting(SearchHit::getId) .containsExactlyInAnyOrder(componentsUuid); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java index 33cc879982a..25b94cbec77 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java @@ -20,12 +20,19 @@ package org.sonar.server.rule.index; import com.google.common.collect.ImmutableMap; +import java.io.IOException; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.indices.AnalyzeRequest; +import org.elasticsearch.client.indices.AnalyzeResponse; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; @@ -93,8 +100,10 @@ public class RuleIndexDefinitionTest { FIELD_RULE_REPOSITORY, "squid", FIELD_RULE_KEY, "squid:S001"))); assertThat(tester.countDocuments(TYPE_RULE)).isEqualTo(1); - assertThat(tester.client().prepareSearch(TYPE_RULE.getIndex()).setQuery(matchQuery(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), "brown fox jumps lazy")) - .get().getHits().getTotalHits()).isEqualTo(1); + assertThat(tester.client().search(EsClient.prepareSearch(TYPE_RULE) + .source(new SearchSourceBuilder() + .query(matchQuery(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), "brown fox jumps lazy")))) + .getHits().getTotalHits()).isEqualTo(new TotalHits(1, TotalHits.Relation.EQUAL_TO)); } @Test @@ -115,9 +124,12 @@ public class RuleIndexDefinitionTest { } private List<AnalyzeResponse.AnalyzeToken> analyzeIndexedTokens(String text) { - return tester.client().nativeClient().admin().indices().prepareAnalyze(TYPE_RULE.getIndex().getName(), - text) - .setField(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION)) - .execute().actionGet().getTokens(); + try { + return tester.nativeClient().indices() + .analyze(AnalyzeRequest.withField(TYPE_RULE.getIndex().getName(), ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), text), RequestOptions.DEFAULT) + .getTokens(); + } catch (IOException e) { + throw new IllegalStateException("Could not analyze indexed tokens for text: " + text); + } } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java index dc0b9bbd56a..b44c1e120e5 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java @@ -29,6 +29,8 @@ import java.util.Set; import java.util.stream.IntStream; import java.util.stream.Stream; import javax.annotation.Nullable; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,6 +47,7 @@ import org.sonar.db.rule.RuleDefinitionDto; import org.sonar.db.rule.RuleDto; import org.sonar.db.rule.RuleDto.Scope; import org.sonar.db.rule.RuleTesting; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.security.SecurityStandards; import org.sonar.server.security.SecurityStandards.SQCategory; diff --git a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java index 86408377761..da9a62d9da3 100644 --- a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java @@ -30,6 +30,7 @@ import java.net.ServerSocket; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -40,18 +41,29 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang.reflect.ConstructorUtils; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.apache.http.HttpHost; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.analysis.common.CommonAnalysisPlugin; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.DiscoveryModule; @@ -59,8 +71,8 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.http.BindHttpException; import org.elasticsearch.http.HttpTransportSettings; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.join.ParentJoinPlugin; @@ -68,6 +80,8 @@ import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.transport.Netty4Plugin; import org.junit.rules.ExternalResource; import org.sonar.api.utils.log.Logger; @@ -87,6 +101,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Lists.newArrayList; import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.sonar.server.es.Index.ALL_INDICES; import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; @@ -114,6 +129,8 @@ public class EsTester extends ExternalResource { } private static final Node SHARED_NODE = createNode(); + private static final EsClient ES_REST_CLIENT = createEsRestClient(SHARED_NODE); + private static final AtomicBoolean CORE_INDICES_CREATED = new AtomicBoolean(false); private static final Set<String> CORE_INDICES_NAMES = new HashSet<>(); @@ -155,26 +172,57 @@ public class EsTester extends ExternalResource { protected void after() { if (isCustom) { // delete non-core indices - String[] existingIndices = SHARED_NODE.client().admin().indices().prepareGetIndex().get().getIndices(); + String[] existingIndices = getIndicesNames(); Stream.of(existingIndices) .filter(i -> !CORE_INDICES_NAMES.contains(i)) .forEach(EsTester::deleteIndexIfExists); } - BulkIndexer.delete(client(), IndexType.main(ALL_INDICES, "dummy"), client().prepareSearch(ALL_INDICES).setQuery(matchAllQuery())); + + BulkIndexer.delete(ES_REST_CLIENT, IndexType.main(ALL_INDICES, "dummy"), + EsClient.prepareSearch(ALL_INDICES.getName()) + .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()))); + } + + private static String[] getIndicesNames() { + String[] existingIndices; + try { + existingIndices = ES_REST_CLIENT.nativeClient().indices().get(new GetIndexRequest(ALL_INDICES.getName()), RequestOptions.DEFAULT).getIndices(); + } catch (ElasticsearchStatusException e) { + if (e.status().getStatus() == 404) { + existingIndices = new String[0]; + } else { + throw e; + } + } catch (IOException e) { + throw new IllegalStateException("Could not get indicies", e); + } + return existingIndices; + } + + private static EsClient createEsRestClient(Node sharedNode) { + assertThat(sharedNode.isClosed()).isFalse(); + + String host = sharedNode.settings().get(HttpTransportSettings.SETTING_HTTP_BIND_HOST.getKey()); + Integer port = sharedNode.settings().getAsInt(HttpTransportSettings.SETTING_HTTP_PORT.getKey(), -1); + return new EsClient(new HttpHost(host, port)); } public EsClient client() { - return new EsClient(SHARED_NODE.client()); + return ES_REST_CLIENT; + } + + public RestHighLevelClient nativeClient() { + return ES_REST_CLIENT.nativeClient(); } public void putDocuments(IndexType indexType, BaseDoc... docs) { try { - BulkRequestBuilder bulk = SHARED_NODE.client().prepareBulk() + BulkRequest bulk = new BulkRequest() .setRefreshPolicy(REFRESH_IMMEDIATE); for (BaseDoc doc : docs) { bulk.add(doc.toIndexRequest()); } - BulkResponse bulkResponse = bulk.get(); + BulkResponse bulkResponse = ES_REST_CLIENT.bulk(bulk); if (bulkResponse.hasFailures()) { throw new IllegalStateException(bulkResponse.buildFailureMessage()); } @@ -185,14 +233,14 @@ public class EsTester extends ExternalResource { public void putDocuments(IndexType indexType, Map<String, Object>... docs) { try { - BulkRequestBuilder bulk = SHARED_NODE.client().prepareBulk() + BulkRequest bulk = new BulkRequest() .setRefreshPolicy(REFRESH_IMMEDIATE); for (Map<String, Object> doc : docs) { IndexType.IndexMainType mainType = indexType.getMainType(); bulk.add(new IndexRequest(mainType.getIndex().getName(), mainType.getType()) .source(doc)); } - BulkResponse bulkResponse = bulk.get(); + BulkResponse bulkResponse = ES_REST_CLIENT.bulk(bulk); if (bulkResponse.hasFailures()) { throw new IllegalStateException(bulkResponse.buildFailureMessage()); } @@ -202,15 +250,23 @@ public class EsTester extends ExternalResource { } public long countDocuments(Index index) { - return client().prepareSearch(index) - .setQuery(matchAllQuery()) - .setSize(0).get().getHits().getTotalHits(); + SearchRequest searchRequest = EsClient.prepareSearch(index.getName()) + .source(new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0)); + + return ES_REST_CLIENT.search(searchRequest) + .getHits().getTotalHits().value; } public long countDocuments(IndexType indexType) { - return client().prepareSearch(indexType.getMainType()) - .setQuery(getDocumentsQuery(indexType)) - .setSize(0).get().getHits().getTotalHits(); + SearchRequest searchRequest = EsClient.prepareSearch(indexType.getMainType()) + .source(new SearchSourceBuilder() + .query(getDocumentsQuery(indexType)) + .size(0)); + + return ES_REST_CLIENT.search(searchRequest) + .getHits().getTotalHits().value; } /** @@ -229,38 +285,33 @@ public class EsTester extends ExternalResource { } /** - * Get all the indexed documents (no paginated results) in the specified index, whatever their type. Results are not sorted. - */ - public List<SearchHit> getDocuments(Index index) { - SearchRequestBuilder req = SHARED_NODE.client() - .prepareSearch(index.getName()) - .setQuery(matchAllQuery()); - return getDocuments(req); - } - - /** * Get all the indexed documents (no paginated results) of the specified type. Results are not sorted. */ public List<SearchHit> getDocuments(IndexType indexType) { IndexType.IndexMainType mainType = indexType.getMainType(); - SearchRequestBuilder req = SHARED_NODE.client() - .prepareSearch(mainType.getIndex().getName()) - .setQuery(getDocumentsQuery(indexType)); - return getDocuments(req); + + SearchRequest searchRequest = EsClient.prepareSearch(mainType.getIndex().getName()) + .source(new SearchSourceBuilder() + .query(getDocumentsQuery(indexType))); + return getDocuments(searchRequest); } - private List<SearchHit> getDocuments(SearchRequestBuilder req) { - EsUtils.optimizeScrollRequest(req); - req.setScroll(new TimeValue(60000)) - .setSize(100); + private List<SearchHit> getDocuments(SearchRequest req) { + req.scroll(new TimeValue(60000)); + req.source() + .size(100) + .sort("_doc", SortOrder.ASC); - SearchResponse response = req.get(); + SearchResponse response = ES_REST_CLIENT.search(req); List<SearchHit> result = newArrayList(); while (true) { Iterables.addAll(result, response.getHits()); - response = SHARED_NODE.client().prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); + response = ES_REST_CLIENT.scroll(new SearchScrollRequest(response.getScrollId()).scroll(new TimeValue(600000))); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(response.getScrollId()); + ES_REST_CLIENT.clearScroll(clearScrollRequest); break; } } @@ -304,19 +355,28 @@ public class EsTester extends ExternalResource { } private void setIndexSettings(String index, Map<String, Object> settings) { - AcknowledgedResponse response = SHARED_NODE.client().admin().indices() - .prepareUpdateSettings(index) - .setSettings(settings) - .get(); + AcknowledgedResponse response = null; + try { + response = ES_REST_CLIENT.nativeClient().indices() + .putSettings(new UpdateSettingsRequest(index).settings(settings), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Could not update index settings", e); + } checkState(response.isAcknowledged()); } private static void deleteIndexIfExists(String name) { try { - AcknowledgedResponse response = SHARED_NODE.client().admin().indices().prepareDelete(name).get(); + AcknowledgedResponse response = ES_REST_CLIENT.nativeClient().indices().delete(new DeleteIndexRequest(name), RequestOptions.DEFAULT); checkState(response.isAcknowledged(), "Fail to drop the index " + name); - } catch (IndexNotFoundException e) { - // ignore + } catch (ElasticsearchStatusException e) { + if (e.status().getStatus() == 404) { + // ignore, index not found + } else { + throw e; + } + } catch (IOException e) { + throw new IllegalStateException("Could not delete index", e); } } @@ -334,28 +394,57 @@ public class EsTester extends ExternalResource { // create index Settings.Builder settings = Settings.builder(); settings.put(index.getSettings()); - CreateIndexResponse indexResponse = SHARED_NODE.client().admin().indices() - .prepareCreate(indexName) - .setSettings(settings) - .get(); + + CreateIndexResponse indexResponse = createIndex(indexName, settings); + if (!indexResponse.isAcknowledged()) { throw new IllegalStateException("Failed to create index " + indexName); } - SHARED_NODE.client().admin().cluster().prepareHealth(indexName).setWaitForStatus(ClusterHealthStatus.YELLOW).get(); + + waitForClusterYellowStatus(indexName); // create types String typeName = index.getMainType().getType(); - AcknowledgedResponse mappingResponse = SHARED_NODE.client().admin().indices().preparePutMapping(indexName) - .setType(typeName) - .setSource(index.getAttributes()) - .get(); + putIndexMapping(index, indexName, typeName); + + waitForClusterYellowStatus(indexName); + + result.add(index); + } + return result; + } + + private static void waitForClusterYellowStatus(String indexName) { + try { + ES_REST_CLIENT.nativeClient().cluster().health(new ClusterHealthRequest(indexName).waitForStatus(ClusterHealthStatus.YELLOW), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Could not query for index health status"); + } + } + + private static void putIndexMapping(BuiltIndex index, String indexName, String typeName) { + try { + AcknowledgedResponse mappingResponse = ES_REST_CLIENT.nativeClient().indices().putMapping(new PutMappingRequest(indexName) + .type(typeName) + .source(index.getAttributes()), RequestOptions.DEFAULT); + if (!mappingResponse.isAcknowledged()) { throw new IllegalStateException("Failed to create type " + typeName); } - SHARED_NODE.client().admin().cluster().prepareHealth(indexName).setWaitForStatus(ClusterHealthStatus.YELLOW).get(); - result.add(index); + } catch (IOException e) { + throw new IllegalStateException("Could not query for put index mapping"); } - return result; + } + + private static CreateIndexResponse createIndex(String indexName, Settings.Builder settings) { + CreateIndexResponse indexResponse; + try { + indexResponse = ES_REST_CLIENT.nativeClient().indices() + .create(new CreateIndexRequest(indexName).settings(settings), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Could not create index"); + } + return indexResponse; } private static Node createNode() { @@ -393,12 +482,11 @@ public class EsTester extends ExternalResource { .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b") // always reduce this - it can make tests really slow .put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.getKey(), TimeValue.timeValueMillis(20)) - .put(NetworkModule.HTTP_ENABLED.getKey(), true) .put(HttpTransportSettings.SETTING_HTTP_PORT.getKey(), httpPort) .put(HttpTransportSettings.SETTING_HTTP_BIND_HOST.getKey(), "localhost") .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "single-node") .build(); - Node node = new Node(InternalSettingsPreparer.prepareEnvironment(settings, null), + Node node = new Node(InternalSettingsPreparer.prepareEnvironment(settings, Collections.emptyMap(), null, null), ImmutableList.of( CommonAnalysisPlugin.class, // Netty4Plugin provides http and tcp transport @@ -406,10 +494,6 @@ public class EsTester extends ExternalResource { // install ParentJoin plugin required to create field of type "join" ParentJoinPlugin.class), true) { - @Override - protected void registerDerivedNodeNameWithLogger(String nodeName) { - // nothing to do - } }; return node.start(); } diff --git a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java index 748e813ae25..7c980ba5b96 100644 --- a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java +++ b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java @@ -19,7 +19,7 @@ */ package org.sonar.server.es.newindex; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.sonar.api.config.internal.MapSettings; import org.sonar.server.es.FakeDoc; import org.sonar.server.es.Index; @@ -47,7 +47,7 @@ public class FakeIndexDefinition implements IndexDefinition { @Override public void define(IndexDefinitionContext context) { NewIndex index = context.create(DESCRIPTOR, newBuilder(new MapSettings().asConfig()).build()); - index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas); + index.getSettings().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas); index.getSettings().put("index.refresh_interval", "-1"); index.createTypeMapping(TYPE_FAKE) .createIntegerField(INT_FIELD); diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java index 8896b4f6869..955798333e9 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java @@ -21,9 +21,7 @@ package org.sonar.server.es; import java.util.Set; import java.util.stream.Collectors; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.sonar.api.config.Configuration; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -108,8 +106,9 @@ public class IndexerStartupTask { } private void waitForIndexYellow(String index) { - Client nativeClient = esClient.nativeClient(); - ClusterHealthAction.INSTANCE.newRequestBuilder(nativeClient).setIndices(index).setWaitForYellowStatus().get(TimeValue.timeValueMinutes(10)); + esClient.clusterHealth(new ClusterHealthRequest() + .indices(index) + .waitForYellowStatus()); } private static String getSynchronousIndexingLogMessage(Set<IndexType> emptyTypes) { diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java index 7d093a2f7e1..9cbafb12d11 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java @@ -26,7 +26,9 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.client.indices.GetIndexRequest; import org.sonar.api.utils.log.Loggers; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.platform.db.migration.es.MigrationEsClient; @@ -41,8 +43,8 @@ public class MigrationEsClientImpl implements MigrationEsClient { @Override public void deleteIndexes(String name, String... otherNames) { - Map<String, IndexStats> indices = client.nativeClient().admin().indices().prepareStats().get().getIndices(); - Set<String> existingIndices = indices.values().stream().map(IndexStats::getIndex).collect(MoreCollectors.toSet()); + String[] indices = client.getIndex(new GetIndexRequest("_all")).getIndices(); + Set<String> existingIndices = Arrays.stream(indices).collect(MoreCollectors.toSet()); Stream.concat(Stream.of(name), Arrays.stream(otherNames)) .distinct() .filter(existingIndices::contains) @@ -51,17 +53,18 @@ public class MigrationEsClientImpl implements MigrationEsClient { @Override public void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType, Map<String, String> options) { - IndexStats stats = client.nativeClient().admin().indices().prepareStats().get().getIndex(index); - if (stats != null) { + String[] indices = client.getIndex(new GetIndexRequest(index)).getIndices(); + if (indices != null && indices.length == 1) { Loggers.get(getClass()).info("Add mapping [{}] to Elasticsearch index [{}]", mappingName, index); String mappingOptions = Stream.concat(Stream.of(Maps.immutableEntry("type", mappingType)), options.entrySet().stream()) .map(e -> e.getKey() + "=" + e.getValue()) .collect(Collectors.joining(",")); - client.nativeClient().admin().indices().preparePutMapping(index) - .setType(type) - .setSource(mappingName, mappingOptions) - .get(); + client.putMapping(new PutMappingRequest(index) + .type(type) + .source(mappingName, mappingOptions)); updatedIndices.add(index); + } else { + throw new IllegalStateException("Expected only one index to be found, actual [" + String.join(",", indices) + "]"); } } @@ -72,6 +75,6 @@ public class MigrationEsClientImpl implements MigrationEsClient { private void deleteIndex(String index) { Loggers.get(getClass()).info("Drop Elasticsearch index [{}]", index); - client.nativeClient().admin().indices().prepareDelete(index).get(); + client.deleteIndex(new DeleteIndexRequest(index)); } } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java index 7d0314480d5..f30781dae33 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java @@ -19,16 +19,15 @@ */ package org.sonar.server.platform.monitoring; -import java.util.Map; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.indices.stats.IndexStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.log.Loggers; import org.sonar.process.systeminfo.Global; import org.sonar.process.systeminfo.SystemInfoSection; import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.IndexStats; +import org.sonar.server.es.response.IndicesStatsResponse; import static org.apache.commons.io.FileUtils.byteCountToDisplaySize; import static org.sonar.process.systeminfo.SystemInfoUtils.setAttribute; @@ -56,12 +55,12 @@ public class EsIndexesSection implements SystemInfoSection, Global { } private void completeIndexAttributes(ProtobufSystemInfo.Section.Builder protobuf) { - IndicesStatsResponse indicesStats = esClient.prepareStats().all().get(); - for (Map.Entry<String, IndexStats> indexStats : indicesStats.getIndices().entrySet()) { - String prefix = "Index " + indexStats.getKey() + " - "; - setAttribute(protobuf, prefix + "Docs", indexStats.getValue().getPrimaries().getDocs().getCount()); - setAttribute(protobuf, prefix + "Shards", indexStats.getValue().getShards().length); - setAttribute(protobuf, prefix + "Store Size", byteCountToDisplaySize(indexStats.getValue().getPrimaries().getStore().getSizeInBytes())); + IndicesStatsResponse indicesStats = esClient.indicesStats(); + for (IndexStats indexStats : indicesStats.getAllIndexStats()) { + String prefix = "Index " + indexStats.getName() + " - "; + setAttribute(protobuf, prefix + "Docs", indexStats.getDocCount()); + setAttribute(protobuf, prefix + "Shards", indexStats.getShardsCount()); + setAttribute(protobuf, prefix + "Store Size", byteCountToDisplaySize(indexStats.getStoreSizeBytes())); } } } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java index ebfa033e44d..aee69ff7b20 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java @@ -21,15 +21,14 @@ package org.sonar.server.platform.monitoring; import java.util.Locale; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.breaker.CircuitBreaker; import org.sonar.api.utils.log.Loggers; import org.sonar.process.systeminfo.SystemInfoSection; import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.NodeStats; +import org.sonar.server.es.response.NodeStatsResponse; import static java.lang.String.format; import static org.apache.commons.io.FileUtils.byteCountToDisplaySize; @@ -44,7 +43,7 @@ public class EsStateSection implements SystemInfoSection { } private ClusterHealthStatus getStateAsEnum() { - return clusterStats().getStatus(); + return esClient.clusterHealth(new ClusterHealthRequest()).getStatus(); } @Override @@ -54,7 +53,7 @@ public class EsStateSection implements SystemInfoSection { try { setAttribute(protobuf, "State", getStateAsEnum().name()); completeNodeAttributes(protobuf); - } catch (Exception es) { + } catch (Exception es) { Loggers.get(EsStateSection.class).warn("Failed to retrieve ES attributes. There will be only a single \"state\" attribute.", es); setAttribute(protobuf, "State", es.getCause() instanceof ElasticsearchException ? es.getCause().getMessage() : es.getMessage()); } @@ -62,42 +61,37 @@ public class EsStateSection implements SystemInfoSection { } private void completeNodeAttributes(ProtobufSystemInfo.Section.Builder protobuf) { - NodesStatsResponse nodesStats = esClient.prepareNodesStats() - .setFs(true) - .setProcess(true) - .setJvm(true) - .setIndices(true) - .setBreaker(true) - .get(); - if (!nodesStats.getNodes().isEmpty()) { - NodeStats stats = nodesStats.getNodes().get(0); - toProtobuf(stats, protobuf); + NodeStatsResponse nodesStatsResponse = esClient.nodesStats(); + + if (!nodesStatsResponse.getNodeStats().isEmpty()) { + toProtobuf(nodesStatsResponse.getNodeStats().get(0), protobuf); } } public static void toProtobuf(NodeStats stats, ProtobufSystemInfo.Section.Builder protobuf) { - setAttribute(protobuf, "CPU Usage (%)", stats.getProcess().getCpu().getPercent()); - setAttribute(protobuf, "Disk Available", byteCountToDisplaySize(stats.getFs().getTotal().getAvailable().getBytes())); - setAttribute(protobuf, "Store Size", byteCountToDisplaySize(stats.getIndices().getStore().getSizeInBytes())); - setAttribute(protobuf, "Translog Size", byteCountToDisplaySize(stats.getIndices().getTranslog().getTranslogSizeInBytes())); - setAttribute(protobuf, "Open File Descriptors", stats.getProcess().getOpenFileDescriptors()); - setAttribute(protobuf, "Max File Descriptors", stats.getProcess().getMaxFileDescriptors()); - setAttribute(protobuf, "JVM Heap Usage", formatPercent(stats.getJvm().getMem().getHeapUsedPercent())); - setAttribute(protobuf, "JVM Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getHeapUsed().getBytes())); - setAttribute(protobuf, "JVM Heap Max", byteCountToDisplaySize(stats.getJvm().getMem().getHeapMax().getBytes())); - setAttribute(protobuf, "JVM Non Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getNonHeapUsed().getBytes())); - setAttribute(protobuf, "JVM Threads", stats.getJvm().getThreads().getCount()); - setAttribute(protobuf, "Field Data Memory", byteCountToDisplaySize(stats.getIndices().getFieldData().getMemorySizeInBytes())); - setAttribute(protobuf, "Field Data Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getLimit())); - setAttribute(protobuf, "Field Data Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated())); - setAttribute(protobuf, "Request Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.REQUEST).getLimit())); - setAttribute(protobuf, "Request Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.REQUEST).getEstimated())); - setAttribute(protobuf, "Query Cache Memory", byteCountToDisplaySize(stats.getIndices().getQueryCache().getMemorySizeInBytes())); - setAttribute(protobuf, "Request Cache Memory", byteCountToDisplaySize(stats.getIndices().getRequestCache().getMemorySizeInBytes())); - } - - private ClusterStatsResponse clusterStats() { - return esClient.prepareClusterStats().get(); + setAttribute(protobuf, "CPU Usage (%)", stats.getCpuUsage()); + setAttribute(protobuf, "Disk Available", byteCountToDisplaySize(stats.getDiskAvailableBytes())); + setAttribute(protobuf, "Store Size", byteCountToDisplaySize(stats.getIndicesStats().getStoreSizeInBytes())); + setAttribute(protobuf, "Translog Size", byteCountToDisplaySize(stats.getIndicesStats().getTranslogSizeInBytes())); + setAttribute(protobuf, "Open File Descriptors", stats.getOpenFileDescriptors()); + setAttribute(protobuf, "Max File Descriptors", stats.getMaxFileDescriptors()); + setAttribute(protobuf, "JVM Heap Usage", formatPercent(stats.getJvmStats().getHeapUsedPercent())); + setAttribute(protobuf, "JVM Heap Used", byteCountToDisplaySize(stats.getJvmStats().getHeapUsedInBytes())); + setAttribute(protobuf, "JVM Heap Max", byteCountToDisplaySize(stats.getJvmStats().getHeapMaxInBytes())); + setAttribute(protobuf, "JVM Non Heap Used", byteCountToDisplaySize(stats.getJvmStats().getNonHeapUsedInBytes())); + setAttribute(protobuf, "JVM Threads", stats.getJvmStats().getThreadCount()); + setAttribute(protobuf, "Field Data Memory", byteCountToDisplaySize(stats.getIndicesStats().getFieldDataMemorySizeInBytes())); + setAttribute(protobuf, "Field Data Circuit Breaker Limit", + byteCountToDisplaySize(stats.getFieldDataCircuitBreakerLimit())); + setAttribute(protobuf, "Field Data Circuit Breaker Estimation", + byteCountToDisplaySize(stats.getFieldDataCircuitBreakerEstimation())); + setAttribute(protobuf, "Request Circuit Breaker Limit", + byteCountToDisplaySize(stats.getRequestCircuitBreakerLimit())); + setAttribute(protobuf, "Request Circuit Breaker Estimation", + byteCountToDisplaySize(stats.getRequestCircuitBreakerEstimation())); + setAttribute(protobuf, "Query Cache Memory", byteCountToDisplaySize(stats.getIndicesStats().getQueryCacheMemorySizeInBytes())); + setAttribute(protobuf, "Request Cache Memory", + byteCountToDisplaySize(stats.getIndicesStats().getRequestCacheMemorySizeInBytes())); } private static String formatPercent(long amount) { diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/EsClusterStateSection.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/EsClusterStateSection.java index 471893b03ee..e44efc3978e 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/EsClusterStateSection.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/EsClusterStateSection.java @@ -19,12 +19,12 @@ */ package org.sonar.server.platform.monitoring.cluster; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; import org.sonar.api.server.ServerSide; import org.sonar.process.systeminfo.Global; import org.sonar.process.systeminfo.SystemInfoSection; import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.ClusterStatsResponse; import static org.sonar.process.systeminfo.SystemInfoUtils.setAttribute; @@ -45,9 +45,9 @@ public class EsClusterStateSection implements SystemInfoSection, Global { public ProtobufSystemInfo.Section toProtobuf() { ProtobufSystemInfo.Section.Builder protobuf = ProtobufSystemInfo.Section.newBuilder(); protobuf.setName("Search State"); - ClusterStatsResponse stats = esClient.prepareClusterStats().get(); - setAttribute(protobuf, "State", stats.getStatus().name()); - setAttribute(protobuf, "Nodes", stats.getNodesStats().getCounts().getTotal()); + ClusterStatsResponse stats = esClient.clusterStats(); + setAttribute(protobuf, "State", stats.getHealthStatus().name()); + setAttribute(protobuf, "Nodes", stats.getNodeCount()); return protobuf.build(); } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImpl.java index 96cf66fc03a..ce0a0c062fd 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImpl.java @@ -22,11 +22,11 @@ package org.sonar.server.platform.monitoring.cluster; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.sonar.api.server.ServerSide; import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.NodeStats; +import org.sonar.server.es.response.NodeStatsResponse; import org.sonar.server.platform.monitoring.EsStateSection; @ServerSide @@ -39,28 +39,22 @@ public class SearchNodesInfoLoaderImpl implements SearchNodesInfoLoader { } public Collection<NodeInfo> load() { - NodesStatsResponse nodesStats = esClient.prepareNodesStats() - .setFs(true) - .setProcess(true) - .setJvm(true) - .setIndices(true) - .setBreaker(true) - .get(); + NodeStatsResponse response = esClient.nodesStats(); + List<NodeInfo> result = new ArrayList<>(); - nodesStats.getNodes().forEach(nodeStat -> result.add(toNodeInfo(nodeStat))); + response.getNodeStats().forEach(nodeStat -> result.add(toNodeInfo(nodeStat))); return result; } private static NodeInfo toNodeInfo(NodeStats stat) { - String nodeName = stat.getNode().getName(); + String nodeName = stat.getName(); NodeInfo info = new NodeInfo(nodeName); - info.setHost(stat.getHostname()); + info.setHost(stat.getHost()); ProtobufSystemInfo.Section.Builder section = ProtobufSystemInfo.Section.newBuilder(); section.setName("Search State"); EsStateSection.toProtobuf(stat, section); info.addSection(section.build()); - return info; } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java index 1746868bb25..f859000c2d2 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java @@ -23,7 +23,8 @@ import com.google.common.collect.ImmutableMap; import java.util.Iterator; import java.util.Map; import javax.annotation.CheckForNull; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.junit.Rule; import org.junit.Test; @@ -33,6 +34,7 @@ import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.platform.db.migration.es.MigrationEsClient; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; public class MigrationEsClientImplTest { @@ -77,8 +79,8 @@ public class MigrationEsClientImplTest { underTest.addMappingToExistingIndex("as", "s", "new_field", "keyword", mappingOptions); assertThat(loadExistingIndices()).toIterable().contains("as"); - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings(); - MappingMetaData mapping = mappings.get("as").get("s"); + ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = mappings(); + MappingMetadata mapping = mappings.get("as").get("s"); assertThat(countMappingFields(mapping)).isEqualTo(1); assertThat(field(mapping, "new_field")).isNotNull(); @@ -87,22 +89,31 @@ public class MigrationEsClientImplTest { assertThat(underTest.getUpdatedIndices()).containsExactly("as"); } + @Test + public void shouldFailIfMoreThanOneIndexReturned() { + String indexPattern = "*s"; + Map<String, String> mappingOptions = ImmutableMap.of("norms", "false"); + assertThatThrownBy(() -> underTest.addMappingToExistingIndex(indexPattern, "s", "new_field", "keyword", mappingOptions)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Expected only one index to be found, actual"); + } + private Iterator<String> loadExistingIndices() { - return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings().keysIt(); + return es.client().getMapping(new GetMappingsRequest()).mappings().keysIt(); } - private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings() { - return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings(); + private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings() { + return es.client().getMapping(new GetMappingsRequest()).mappings(); } @CheckForNull @SuppressWarnings("unchecked") - private Map<String, Object> field(MappingMetaData mapping, String field) { + private Map<String, Object> field(MappingMetadata mapping, String field) { Map<String, Object> props = (Map<String, Object>) mapping.getSourceAsMap().get("properties"); return (Map<String, Object>) props.get(field); } - private int countMappingFields(MappingMetaData mapping) { + private int countMappingFields(MappingMetadata mapping) { return ((Map) mapping.getSourceAsMap().get("properties")).size(); } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsIndexesSectionTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsIndexesSectionTest.java index e340ecf56c4..43559c7dbfe 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsIndexesSectionTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsIndexesSectionTest.java @@ -27,6 +27,7 @@ import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.sonar.process.systeminfo.SystemInfoUtils.attribute; @@ -58,7 +59,7 @@ public class EsIndexesSectionTest { public void attributes_displays_exception_message_when_cause_null_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsIndexesSection underTest = new EsIndexesSection(esClientMock); - when(esClientMock.prepareStats()).thenThrow(new RuntimeException("RuntimeException with no cause")); + when(esClientMock.indicesStats()).thenThrow(new RuntimeException("RuntimeException with no cause")); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "Error", "RuntimeException with no cause"); @@ -68,7 +69,7 @@ public class EsIndexesSectionTest { public void attributes_displays_exception_message_when_cause_is_not_ElasticSearchException_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsIndexesSection underTest = new EsIndexesSection(esClientMock); - when(esClientMock.prepareStats()).thenThrow(new RuntimeException("RuntimeException with cause not ES", new IllegalArgumentException("some cause message"))); + when(esClientMock.indicesStats()).thenThrow(new RuntimeException("RuntimeException with cause not ES", new IllegalArgumentException("some cause message"))); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "Error", "RuntimeException with cause not ES"); @@ -78,7 +79,7 @@ public class EsIndexesSectionTest { public void attributes_displays_cause_message_when_cause_is_ElasticSearchException_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsIndexesSection underTest = new EsIndexesSection(esClientMock); - when(esClientMock.prepareStats()).thenThrow(new RuntimeException("RuntimeException with ES cause", new ElasticsearchException("some cause message"))); + when(esClientMock.indicesStats()).thenThrow(new RuntimeException("RuntimeException with ES cause", new ElasticsearchException("some cause message"))); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "Error", "some cause message"); diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsStateSectionTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsStateSectionTest.java index f0445e2f831..d7da13bf907 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsStateSectionTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsStateSectionTest.java @@ -28,6 +28,7 @@ import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.sonar.process.systeminfo.SystemInfoUtils.attribute; @@ -63,7 +64,7 @@ public class EsStateSectionTest { public void attributes_displays_exception_message_when_cause_null_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsStateSection underTest = new EsStateSection(esClientMock); - when(esClientMock.prepareClusterStats()).thenThrow(new RuntimeException("RuntimeException with no cause")); + when(esClientMock.clusterHealth(any())).thenThrow(new RuntimeException("RuntimeException with no cause")); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "State", "RuntimeException with no cause"); @@ -73,7 +74,7 @@ public class EsStateSectionTest { public void attributes_displays_exception_message_when_cause_is_not_ElasticSearchException_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsStateSection underTest = new EsStateSection(esClientMock); - when(esClientMock.prepareClusterStats()).thenThrow(new RuntimeException("RuntimeException with cause not ES", new IllegalArgumentException("some cause message"))); + when(esClientMock.clusterHealth(any())).thenThrow(new RuntimeException("RuntimeException with cause not ES", new IllegalArgumentException("some cause message"))); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "State", "RuntimeException with cause not ES"); @@ -83,7 +84,7 @@ public class EsStateSectionTest { public void attributes_displays_cause_message_when_cause_is_ElasticSearchException_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsStateSection underTest = new EsStateSection(esClientMock); - when(esClientMock.prepareClusterStats()).thenThrow(new RuntimeException("RuntimeException with ES cause", new ElasticsearchException("some cause message"))); + when(esClientMock.clusterHealth(any())).thenThrow(new RuntimeException("RuntimeException with ES cause", new ElasticsearchException("some cause message"))); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "State", "some cause message"); diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java index ad80fe587e0..489e18e758b 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.function.Consumer; import java.util.stream.Stream; import javax.annotation.Nullable; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -35,10 +35,10 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; -import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters; -import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters.InternalBucket; -import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits; -import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters; +import org.elasticsearch.search.aggregations.metrics.ParsedTopHits; +import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; @@ -81,11 +81,10 @@ public class ComponentIndex { } public SearchIdResult<String> search(ComponentQuery query, SearchOptions searchOptions) { - SearchRequestBuilder requestBuilder = client - .prepareSearch(TYPE_COMPONENT.getMainType()) - .setFetchSource(false) - .setFrom(searchOptions.getOffset()) - .setSize(searchOptions.getLimit()); + SearchSourceBuilder source = new SearchSourceBuilder() + .fetchSource(false) + .from(searchOptions.getOffset()) + .size(searchOptions.getLimit()); BoolQueryBuilder esQuery = boolQuery(); esQuery.filter(authorizationTypeSupport.createQueryFilter()); @@ -99,10 +98,13 @@ public class ComponentIndex { }); setEmptiable(query.getQualifiers(), q -> esQuery.must(termsQuery(FIELD_QUALIFIER, q))); setNullable(query.getOrganizationUuid(), o -> esQuery.must(termQuery(FIELD_ORGANIZATION_UUID, o))); - requestBuilder.setQuery(esQuery); - requestBuilder.addSort(SORTABLE_ANALYZER.subField(FIELD_NAME), SortOrder.ASC); + source.sort(SORTABLE_ANALYZER.subField(FIELD_NAME), SortOrder.ASC); - return new SearchIdResult<>(requestBuilder.get(), id -> id, system2.getDefaultTimeZone()); + source.query(esQuery); + + SearchRequest request = EsClient.prepareSearch(TYPE_COMPONENT.getMainType()) + .source(source); + return new SearchIdResult<>(client.search(request), id -> id, system2.getDefaultTimeZone()); } public ComponentIndexResults searchSuggestions(SuggestionQuery query) { @@ -116,15 +118,16 @@ public class ComponentIndex { return ComponentIndexResults.newBuilder().build(); } - SearchRequestBuilder request = client - .prepareSearch(TYPE_COMPONENT.getMainType()) - .setQuery(createQuery(query, features)) - .addAggregation(createAggregation(query)) + SearchSourceBuilder source = new SearchSourceBuilder() + .query(createQuery(query, features)) + .aggregation(createAggregation(query)) // the search hits are part of the aggregations - .setSize(0); + .size(0); - SearchResponse response = request.get(); + SearchRequest request = EsClient.prepareSearch(TYPE_COMPONENT.getMainType()) + .source(source); + SearchResponse response = client.search(request); return aggregationsToQualifiers(response); } @@ -178,21 +181,21 @@ public class ComponentIndex { } private static ComponentIndexResults aggregationsToQualifiers(SearchResponse response) { - InternalFilters filtersAgg = response.getAggregations().get(FILTERS_AGGREGATION_NAME); - List<InternalBucket> buckets = filtersAgg.getBuckets(); + ParsedFilters filtersAgg = response.getAggregations().get(FILTERS_AGGREGATION_NAME); + List<ParsedFilters.ParsedBucket> buckets = (List<ParsedFilters.ParsedBucket>) filtersAgg.getBuckets(); return ComponentIndexResults.newBuilder() .setQualifiers( buckets.stream().map(ComponentIndex::bucketToQualifier)) .build(); } - private static ComponentHitsPerQualifier bucketToQualifier(InternalBucket bucket) { - InternalTopHits docs = bucket.getAggregations().get(DOCS_AGGREGATION_NAME); + private static ComponentHitsPerQualifier bucketToQualifier(ParsedFilters.ParsedBucket bucket) { + ParsedTopHits docs = bucket.getAggregations().get(DOCS_AGGREGATION_NAME); SearchHits hitList = docs.getHits(); SearchHit[] hits = hitList.getHits(); - return new ComponentHitsPerQualifier(bucket.getKey(), ComponentHit.fromSearchHits(hits), hitList.getTotalHits()); + return new ComponentHitsPerQualifier(bucket.getKey(), ComponentHit.fromSearchHits(hits), hitList.getTotalHits().value); } private static <T> void setNullable(@Nullable T parameter, Consumer<T> consumer) { diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java index e671e14a38f..9f16edab51c 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java @@ -25,10 +25,14 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.picocontainer.Startable; @@ -78,7 +82,7 @@ public class IndexCreator implements Startable { public void start() { // create the "metadata" index first IndexType.IndexMainType metadataMainType = TYPE_METADATA; - if (!client.prepareIndicesExist(metadataMainType.getIndex()).get().isExists()) { + if (!client.indexExists(new GetIndexRequest(metadataMainType.getIndex().getName()))) { IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); metadataIndexDefinition.define(context); NewIndex index = context.getIndices().values().iterator().next(); @@ -93,7 +97,7 @@ public class IndexCreator implements Startable { definitions.getIndices().values().stream() .filter(i -> !i.getMainType().equals(metadataMainType)) .forEach(index -> { - boolean exists = client.prepareIndicesExist(index.getMainType().getIndex()).get().isExists(); + boolean exists = client.indexExists(new GetIndexRequest(index.getMainType().getIndex().getName())); if (!exists) { createIndex(index, true); } else if (hasDefinitionChange(index)) { @@ -112,7 +116,7 @@ public class IndexCreator implements Startable { private boolean isReadOnly(IndexType.IndexMainType mainType) { String indexName = mainType.getIndex().getName(); - String readOnly = client.nativeClient().admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet() + String readOnly = client.getSettings(new GetSettingsRequest().indices(indexName)) .getSetting(indexName, "index.blocks.read_only_allow_delete"); return "true".equalsIgnoreCase(readOnly); } @@ -123,9 +127,8 @@ public class IndexCreator implements Startable { String indexName = mainType.getIndex().getName(); Settings.Builder builder = Settings.builder(); builder.putNull("index.blocks.read_only_allow_delete"); - client.nativeClient().admin().indices() - .updateSettings(new UpdateSettingsRequest().indices(indexName).settings(builder.build())) - .actionGet(); + + client.putSettings(new UpdateSettingsRequest().indices(indexName).settings(builder.build())); } @Override @@ -143,10 +146,8 @@ public class IndexCreator implements Startable { metadataIndex.setInitialized(builtIndex.getMainType(), false); builtIndex.getRelationTypes().forEach(relationType -> metadataIndex.setInitialized(relationType, false)); } - CreateIndexResponse indexResponse = client - .prepareCreate(index) - .setSettings(settings) - .get(); + CreateIndexResponse indexResponse = client.create(new CreateIndexRequest(index.getName()).settings((settings))); + if (!indexResponse.isAcknowledged()) { throw new IllegalStateException("Failed to create index [" + index.getName() + "]"); } @@ -154,10 +155,10 @@ public class IndexCreator implements Startable { // create types LOGGER.info("Create type {}", builtIndex.getMainType().format()); - AcknowledgedResponse mappingResponse = client.preparePutMapping(index) - .setType(builtIndex.getMainType().getType()) - .setSource(builtIndex.getAttributes()) - .get(); + AcknowledgedResponse mappingResponse = client.putMapping(new PutMappingRequest(builtIndex.getMainType().getIndex().getName()) + .type(builtIndex.getMainType().getType()) + .source(builtIndex.getAttributes())); + if (!mappingResponse.isAcknowledged()) { throw new IllegalStateException("Failed to create type " + builtIndex.getMainType().getType()); } @@ -165,7 +166,7 @@ public class IndexCreator implements Startable { } private void deleteIndex(String indexName) { - client.nativeClient().admin().indices().prepareDelete(indexName).get(); + client.deleteIndex(new DeleteIndexRequest(indexName)); } private void updateIndex(BuiltIndex<?> index) { @@ -215,7 +216,7 @@ public class IndexCreator implements Startable { Set<String> definedNames = definitions.stream() .map(t -> t.getMainType().getIndex().getName()) .collect(Collectors.toSet()); - return Arrays.stream(client.nativeClient().admin().indices().prepareGetIndex().get().getIndices()) + return Arrays.stream(client.getIndex(new GetIndexRequest("_all")).getIndices()) .filter(definedNames::contains) .filter(index -> !DESCRIPTOR.getName().equals(index)) .collect(Collectors.toList()); diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 9afa4787e17..1446e48f4ab 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -38,7 +38,7 @@ import java.util.stream.Stream; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -48,21 +48,21 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.HasAggregations; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; +import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; -import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms; -import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.max.InternalMax; -import org.elasticsearch.search.aggregations.metrics.min.Min; -import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount; +import org.elasticsearch.search.aggregations.metrics.Min; +import org.elasticsearch.search.aggregations.metrics.ParsedMax; +import org.elasticsearch.search.aggregations.metrics.ParsedValueCount; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; -import org.joda.time.DateTimeZone; import org.joda.time.Duration; import org.sonar.api.issue.Issue; import org.sonar.api.rule.Severity; @@ -88,6 +88,7 @@ import org.sonar.server.issue.index.IssueQuery.PeriodStart; import org.sonar.server.permission.index.AuthorizationDoc; import org.sonar.server.permission.index.WebAuthorizationTypeSupport; import org.sonar.server.security.SecurityStandards; +import org.sonar.server.security.SecurityStandards.SQCategory; import org.sonar.server.user.UserSession; import org.sonar.server.view.index.ViewIndexDefinition; @@ -344,39 +345,41 @@ public class IssueIndex { } public SearchResponse search(IssueQuery query, SearchOptions options) { - SearchRequestBuilder esRequest = client.prepareSearch(TYPE_ISSUE.getMainType()); + SearchRequest requestBuilder = EsClient.prepareSearch(TYPE_ISSUE.getMainType()); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + requestBuilder.source(sourceBuilder); - configureSorting(query, esRequest); - configurePagination(options, esRequest); - configureRouting(query, options, esRequest); + configureSorting(query, sourceBuilder); + configurePagination(options, sourceBuilder); + configureRouting(query, options, requestBuilder); AllFilters allFilters = createAllFilters(query); RequestFiltersComputer filterComputer = newFilterComputer(options, allFilters); - configureTopAggregations(query, options, esRequest, allFilters, filterComputer); - configureQuery(esRequest, filterComputer); - configureTopFilters(esRequest, filterComputer); + configureTopAggregations(query, options, sourceBuilder, allFilters, filterComputer); + configureQuery(sourceBuilder, filterComputer); + configureTopFilters(sourceBuilder, filterComputer); - esRequest.setFetchSource(false); + sourceBuilder.fetchSource(false); - return esRequest.get(); + return client.search(requestBuilder); } - private void configureTopAggregations(IssueQuery query, SearchOptions options, SearchRequestBuilder esRequest, AllFilters allFilters, RequestFiltersComputer filterComputer) { + private void configureTopAggregations(IssueQuery query, SearchOptions options, SearchSourceBuilder esRequest, AllFilters allFilters, RequestFiltersComputer filterComputer) { TopAggregationHelper aggregationHelper = newAggregationHelper(filterComputer, query); configureTopAggregations(aggregationHelper, query, options, allFilters, esRequest); } - private static void configureQuery(SearchRequestBuilder esRequest, RequestFiltersComputer filterComputer) { + private static void configureQuery(SearchSourceBuilder esRequest, RequestFiltersComputer filterComputer) { QueryBuilder esQuery = filterComputer.getQueryFilters() .map(t -> (QueryBuilder) boolQuery().must(matchAllQuery()).filter(t)) .orElse(matchAllQuery()); - esRequest.setQuery(esQuery); + esRequest.query(esQuery); } - private static void configureTopFilters(SearchRequestBuilder esRequest, RequestFiltersComputer filterComputer) { - filterComputer.getPostFilters().ifPresent(esRequest::setPostFilter); + private static void configureTopFilters(SearchSourceBuilder esRequest, RequestFiltersComputer filterComputer) { + filterComputer.getPostFilters().ifPresent(esRequest::postFilter); } /** @@ -386,15 +389,15 @@ public class IssueIndex { * Note that sticky facets may involve all projects, so this optimization must be * disabled when facets are enabled. */ - private static void configureRouting(IssueQuery query, SearchOptions options, SearchRequestBuilder requestBuilder) { + private static void configureRouting(IssueQuery query, SearchOptions options, SearchRequest searchRequest) { Collection<String> uuids = query.projectUuids(); if (!uuids.isEmpty() && options.getFacets().isEmpty()) { - requestBuilder.setRouting(uuids.stream().map(AuthorizationDoc::idOf).toArray(String[]::new)); + searchRequest.routing(uuids.stream().map(AuthorizationDoc::idOf).toArray(String[]::new)); } } - private static void configurePagination(SearchOptions options, SearchRequestBuilder esSearch) { - esSearch.setFrom(options.getOffset()).setSize(options.getLimit()); + private static void configurePagination(SearchOptions options, SearchSourceBuilder esSearch) { + esSearch.from(options.getOffset()).size(options.getLimit()); } private AllFilters createAllFilters(IssueQuery query) { @@ -596,8 +599,8 @@ public class IssueIndex { return value == null ? null : termQuery(field, value); } - private void configureSorting(IssueQuery query, SearchRequestBuilder esRequest) { - createSortBuilders(query).forEach(esRequest::addSort); + private void configureSorting(IssueQuery query, SearchSourceBuilder esRequest) { + createSortBuilders(query).forEach(esRequest::sort); } private List<FieldSortBuilder> createSortBuilders(IssueQuery query) { @@ -659,7 +662,7 @@ public class IssueIndex { } private void configureTopAggregations(TopAggregationHelper aggregationHelper, IssueQuery query, SearchOptions options, - AllFilters queryFilters, SearchRequestBuilder esRequest) { + AllFilters queryFilters, SearchSourceBuilder esRequest) { addFacetIfNeeded(options, aggregationHelper, esRequest, STATUSES, NO_SELECTED_VALUES); addFacetIfNeeded(options, aggregationHelper, esRequest, PROJECT_UUIDS, query.projectUuids().toArray()); addFacetIfNeeded(options, aggregationHelper, esRequest, MODULE_UUIDS, query.moduleUuids().toArray()); @@ -687,7 +690,7 @@ public class IssueIndex { } private static void addFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, - SearchRequestBuilder esRequest, Facet facet, Object[] selectedValues) { + SearchSourceBuilder esRequest, Facet facet, Object[] selectedValues) { if (!options.getFacets().contains(facet.getName())) { return; } @@ -697,10 +700,10 @@ public class IssueIndex { NO_EXTRA_FILTER, t -> aggregationHelper.getSubAggregationHelper().buildSelectedItemsAggregation(facet.getName(), facet.getTopAggregationDef(), selectedValues) .ifPresent(t::subAggregation)); - esRequest.addAggregation(topAggregation); + esRequest.aggregation(topAggregation); } - private static void addSecurityCategoryFacetIfNeeded(String param, Facet facet, SearchOptions options, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest, + private static void addSecurityCategoryFacetIfNeeded(String param, Facet facet, SearchOptions options, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest, Object[] selectedValues) { if (!options.getFacets().contains(param)) { return; @@ -711,10 +714,10 @@ public class IssueIndex { filter -> filter.must(termQuery(FIELD_ISSUE_TYPE, VULNERABILITY.name())), t -> aggregationHelper.getSubAggregationHelper().buildSelectedItemsAggregation(facet.getName(), facet.getTopAggregationDef(), selectedValues) .ifPresent(t::subAggregation)); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } - private static void addSeverityFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private static void addSeverityFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { if (!options.getFacets().contains(PARAM_SEVERITIES)) { return; } @@ -724,10 +727,10 @@ public class IssueIndex { // Ignore severity of Security HotSpots filter -> filter.mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name())), NO_OTHER_SUBAGGREGATION); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } - private static void addResolutionFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private static void addResolutionFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { if (!options.getFacets().contains(PARAM_RESOLUTIONS)) { return; } @@ -742,10 +745,10 @@ public class IssueIndex { .missing(RESOLUTIONS.getName() + FACET_SUFFIX_MISSING) .field(RESOLUTIONS.getFieldName()))); }); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } - private static void addAssigneesFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private static void addAssigneesFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { if (!options.getFacets().contains(PARAM_ASSIGNEES)) { return; } @@ -765,13 +768,13 @@ public class IssueIndex { AggregationBuilder aggregation = aggregationHelper.buildTermTopAggregation( ASSIGNEES.getName(), ASSIGNEES.getTopAggregationDef(), ASSIGNEES.getNumberOfTerms(), NO_EXTRA_FILTER, assigneeAggregations); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } private void addCreatedAtFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, AllFilters allFilters, - SearchRequestBuilder esRequest) { + SearchSourceBuilder esRequest) { if (options.getFacets().contains(PARAM_CREATED_AT)) { - getCreatedAtFacet(query, aggregationHelper, allFilters).ifPresent(esRequest::addAggregation); + getCreatedAtFacet(query, aggregationHelper, allFilters).ifPresent(esRequest::aggregation); } } @@ -806,7 +809,7 @@ public class IssueIndex { .dateHistogramInterval(bucketSize) .minDocCount(0L) .format(DateUtils.DATETIME_FORMAT) - .timeZone(DateTimeZone.forOffsetMillis(system.getDefaultTimeZone().getRawOffset())) + .timeZone(system.getDefaultTimeZone().toZoneId()) // ES dateHistogram bounds are inclusive while createdBefore parameter is exclusive .extendedBounds(new ExtendedBounds(startInclusive ? startTime : (startTime + 1), endTime - 1L)); addEffortAggregationIfNeeded(query, dateHistogram); @@ -831,17 +834,20 @@ public class IssueIndex { private OptionalLong getMinCreatedAt(AllFilters filters) { String facetNameAndField = CREATED_AT.getFieldName(); - SearchRequestBuilder esRequest = client - .prepareSearch(TYPE_ISSUE.getMainType()) - .setSize(0); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .size(0); BoolQueryBuilder esFilter = boolQuery(); filters.stream().filter(Objects::nonNull).forEach(esFilter::must); if (esFilter.hasClauses()) { - esRequest.setQuery(QueryBuilders.boolQuery().filter(esFilter)); + sourceBuilder.query(QueryBuilders.boolQuery().filter(esFilter)); } - esRequest.addAggregation(AggregationBuilders.min(facetNameAndField).field(facetNameAndField)); - Min minValue = esRequest.get().getAggregations().get(facetNameAndField); + sourceBuilder.aggregation(AggregationBuilders.min(facetNameAndField).field(facetNameAndField)); + + SearchRequest request = EsClient.prepareSearch(TYPE_ISSUE.getMainType()) + .source(sourceBuilder); + Min minValue = client.search(request).getAggregations().get(facetNameAndField); double actualValue = minValue.getValue(); if (Double.isInfinite(actualValue)) { return OptionalLong.empty(); @@ -849,7 +855,7 @@ public class IssueIndex { return OptionalLong.of((long) actualValue); } - private void addAssignedToMeFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private void addAssignedToMeFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { String uuid = userSession.getUuid(); if (options.getFacets().contains(ASSIGNED_TO_ME.getName()) && !StringUtils.isEmpty(uuid)) { AggregationBuilder aggregation = aggregationHelper.buildTermTopAggregation( @@ -863,17 +869,17 @@ public class IssueIndex { .buildSelectedItemsAggregation(ASSIGNED_TO_ME.getName(), ASSIGNED_TO_ME.getTopAggregationDef(), new String[] {uuid}) .ifPresent(t::subAggregation); }); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } } - private static void addEffortTopAggregation(TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private static void addEffortTopAggregation(TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { AggregationBuilder topAggregation = aggregationHelper.buildTopAggregation( FACET_MODE_EFFORT, EFFORT_TOP_AGGREGATION, NO_EXTRA_FILTER, t -> t.subAggregation(EFFORT_AGGREGATION)); - esRequest.addAggregation(topAggregation); + esRequest.aggregation(topAggregation); } public List<String> searchTags(IssueQuery query, @Nullable String textQuery, int size) { @@ -892,12 +898,14 @@ public class IssueIndex { } private Terms listTermsMatching(String fieldName, IssueQuery query, @Nullable String textQuery, BucketOrder termsOrder, int size) { - SearchRequestBuilder requestBuilder = client - .prepareSearch(TYPE_ISSUE.getMainType()) + SearchRequest requestBuilder = EsClient.prepareSearch(TYPE_ISSUE.getMainType()); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() // Avoids returning search hits - .setSize(0); + .size(0); + requestBuilder.source(sourceBuilder); - requestBuilder.setQuery(boolQuery().must(QueryBuilders.matchAllQuery()).filter(createBoolFilter(query))); + sourceBuilder.query(boolQuery().must(QueryBuilders.matchAllQuery()).filter(createBoolFilter(query))); TermsAggregationBuilder aggreg = AggregationBuilders.terms("_ref") .field(fieldName) @@ -908,7 +916,9 @@ public class IssueIndex { aggreg.includeExclude(new IncludeExclude(format(SUBSTRING_MATCH_REGEXP, escapeSpecialRegexChars(textQuery)), null)); } - SearchResponse searchResponse = requestBuilder.addAggregation(aggreg).get(); + sourceBuilder.aggregation(aggreg); + + SearchResponse searchResponse = client.search(requestBuilder); return searchResponse.getAggregations().get("_ref"); } @@ -926,18 +936,19 @@ public class IssueIndex { if (projectUuids.isEmpty()) { return Collections.emptyList(); } - SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType()) - .setQuery( + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query( boolQuery() .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)) .filter(termQuery(FIELD_ISSUE_ASSIGNEE_UUID, assigneeUuid)) .mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name()))) - .setSize(0); + .size(0); + IntStream.range(0, projectUuids.size()).forEach(i -> { String projectUuid = projectUuids.get(i); long from = froms.get(i); - request - .addAggregation(AggregationBuilders + sourceBuilder + .aggregation(AggregationBuilders .filter(projectUuid, boolQuery() .filter(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid)) .filter(rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT).gte(epochMillisToEpochSeconds(from)))) @@ -948,16 +959,20 @@ public class IssueIndex { .subAggregation( AggregationBuilders.max("maxFuncCreatedAt").field(FIELD_ISSUE_FUNC_CREATED_AT)))); }); - SearchResponse response = request.get(); + + SearchRequest requestBuilder = EsClient.prepareSearch(TYPE_ISSUE.getMainType()); + + requestBuilder.source(sourceBuilder); + SearchResponse response = client.search(requestBuilder); return response.getAggregations().asList().stream() - .map(x -> (InternalFilter) x) - .flatMap(projectBucket -> ((StringTerms) projectBucket.getAggregations().get("branchUuid")).getBuckets().stream() + .map(x -> (ParsedFilter) x) + .flatMap(projectBucket -> ((ParsedStringTerms) projectBucket.getAggregations().get("branchUuid")).getBuckets().stream() .flatMap(branchBucket -> { - long count = ((InternalValueCount) branchBucket.getAggregations().get(AGG_COUNT)).getValue(); + long count = ((ParsedValueCount) branchBucket.getAggregations().get(AGG_COUNT)).getValue(); if (count < 1L) { return Stream.empty(); } - long lastIssueDate = (long) ((InternalMax) branchBucket.getAggregations().get("maxFuncCreatedAt")).getValue(); + long lastIssueDate = (long) ((ParsedMax) branchBucket.getAggregations().get("maxFuncCreatedAt")).getValue(); return Stream.of(new ProjectStatistics(branchBucket.getKeyAsString(), count, lastIssueDate)); })) .collect(MoreCollectors.toList(projectUuids.size())); @@ -968,32 +983,36 @@ public class IssueIndex { return Collections.emptyList(); } - SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType()) - .setRouting(AuthorizationDoc.idOf(projectUuid)) - .setQuery( + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query( boolQuery() .must(termsQuery(FIELD_ISSUE_BRANCH_UUID, branchUuids)) .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)) .must(termQuery(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(false)))) - .setSize(0) - .addAggregation(AggregationBuilders.terms("branchUuids") + .size(0) + .aggregation(AggregationBuilders.terms("branchUuids") .field(FIELD_ISSUE_BRANCH_UUID) .size(branchUuids.size()) .subAggregation(AggregationBuilders.terms("types") .field(FIELD_ISSUE_TYPE))); - SearchResponse response = request.get(); - return ((StringTerms) response.getAggregations().get("branchUuids")).getBuckets().stream() + + SearchRequest requestBuilder = EsClient.prepareSearch(TYPE_ISSUE.getMainType()) + .routing(AuthorizationDoc.idOf(projectUuid)); + + requestBuilder.source(sourceBuilder); + SearchResponse response = client.search(requestBuilder); + return ((ParsedStringTerms) response.getAggregations().get("branchUuids")).getBuckets().stream() .map(bucket -> new PrStatistics(bucket.getKeyAsString(), - ((StringTerms) bucket.getAggregations().get("types")).getBuckets() + ((ParsedStringTerms) bucket.getAggregations().get("types")).getBuckets() .stream() - .collect(uniqueIndex(StringTerms.Bucket::getKeyAsString, InternalTerms.Bucket::getDocCount)))) + .collect(uniqueIndex(MultiBucketsAggregation.Bucket::getKeyAsString, MultiBucketsAggregation.Bucket::getDocCount)))) .collect(MoreCollectors.toList(branchUuids.size())); } public List<SecurityStandardCategoryStatistics> getSansTop25Report(String projectUuid, boolean isViewOrApp, boolean includeCwe) { - SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); + SearchSourceBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); Stream.of(SANS_TOP_25_INSECURE_INTERACTION, SANS_TOP_25_RISKY_RESOURCE, SANS_TOP_25_POROUS_DEFENSES) - .forEach(sansCategory -> request.addAggregation(newSecurityReportSubAggregations( + .forEach(sansCategory -> request.aggregation(newSecurityReportSubAggregations( AggregationBuilders.filter(sansCategory, boolQuery().filter(termQuery(FIELD_ISSUE_SANS_TOP_25, sansCategory))), includeCwe, Optional.ofNullable(SecurityStandards.CWES_BY_SANS_TOP_25.get(sansCategory))))); @@ -1001,9 +1020,9 @@ public class IssueIndex { } public List<SecurityStandardCategoryStatistics> getSonarSourceReport(String projectUuid, boolean isViewOrApp, boolean includeCwe) { - SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); - Arrays.stream(SecurityStandards.SQCategory.values()) - .forEach(sonarsourceCategory -> request.addAggregation( + SearchSourceBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); + Arrays.stream(SQCategory.values()) + .forEach(sonarsourceCategory -> request.aggregation( newSecurityReportSubAggregations( AggregationBuilders.filter(sonarsourceCategory.getKey(), boolQuery().filter(termQuery(FIELD_ISSUE_SQ_SECURITY_CATEGORY, sonarsourceCategory.getKey()))), includeCwe, @@ -1012,9 +1031,9 @@ public class IssueIndex { } public List<SecurityStandardCategoryStatistics> getOwaspTop10Report(String projectUuid, boolean isViewOrApp, boolean includeCwe) { - SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); + SearchSourceBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); IntStream.rangeClosed(1, 10).mapToObj(i -> "a" + i) - .forEach(owaspCategory -> request.addAggregation( + .forEach(owaspCategory -> request.aggregation( newSecurityReportSubAggregations( AggregationBuilders.filter(owaspCategory, boolQuery().filter(termQuery(FIELD_ISSUE_OWASP_TOP_10, owaspCategory))), includeCwe, @@ -1022,17 +1041,19 @@ public class IssueIndex { return processSecurityReportSearchResults(request, includeCwe); } - private static List<SecurityStandardCategoryStatistics> processSecurityReportSearchResults(SearchRequestBuilder request, boolean includeCwe) { - SearchResponse response = request.get(); + private List<SecurityStandardCategoryStatistics> processSecurityReportSearchResults(SearchSourceBuilder sourceBuilder, boolean includeCwe) { + SearchRequest request = EsClient.prepareSearch(TYPE_ISSUE.getMainType()) + .source(sourceBuilder); + SearchResponse response = client.search(request); return response.getAggregations().asList().stream() - .map(c -> processSecurityReportIssueSearchResults((InternalFilter) c, includeCwe)) + .map(c -> processSecurityReportIssueSearchResults((ParsedFilter) c, includeCwe)) .collect(MoreCollectors.toList()); } - private static SecurityStandardCategoryStatistics processSecurityReportIssueSearchResults(InternalFilter categoryBucket, boolean includeCwe) { + private static SecurityStandardCategoryStatistics processSecurityReportIssueSearchResults(ParsedFilter categoryBucket, boolean includeCwe) { List<SecurityStandardCategoryStatistics> children = new ArrayList<>(); if (includeCwe) { - Stream<StringTerms.Bucket> stream = ((StringTerms) categoryBucket.getAggregations().get(AGG_CWES)).getBuckets().stream(); + Stream<? extends Terms.Bucket> stream = ((ParsedStringTerms) categoryBucket.getAggregations().get(AGG_CWES)).getBuckets().stream(); children = stream.map(cweBucket -> processSecurityReportCategorySearchResults(cweBucket, cweBucket.getKeyAsString(), null)).collect(toList()); } @@ -1041,18 +1062,18 @@ public class IssueIndex { private static SecurityStandardCategoryStatistics processSecurityReportCategorySearchResults(HasAggregations categoryBucket, String categoryName, @Nullable List<SecurityStandardCategoryStatistics> children) { - List<StringTerms.Bucket> severityBuckets = ((StringTerms) ((InternalFilter) categoryBucket.getAggregations().get(AGG_VULNERABILITIES)).getAggregations().get(AGG_SEVERITIES)) - .getBuckets(); - long vulnerabilities = severityBuckets.stream().mapToLong(b -> ((InternalValueCount) b.getAggregations().get(AGG_COUNT)).getValue()).sum(); + List<? extends Terms.Bucket> severityBuckets = ((ParsedStringTerms) ((ParsedFilter) categoryBucket.getAggregations().get(AGG_VULNERABILITIES)).getAggregations() + .get(AGG_SEVERITIES)).getBuckets(); + long vulnerabilities = severityBuckets.stream().mapToLong(b -> ((ParsedValueCount) b.getAggregations().get(AGG_COUNT)).getValue()).sum(); // Worst severity having at least one issue OptionalInt severityRating = severityBuckets.stream() - .filter(b -> ((InternalValueCount) b.getAggregations().get(AGG_COUNT)).getValue() != 0) + .filter(b -> ((ParsedValueCount) b.getAggregations().get(AGG_COUNT)).getValue() != 0) .mapToInt(b -> Severity.ALL.indexOf(b.getKeyAsString()) + 1) .max(); - long toReviewSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get(AGG_TO_REVIEW_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) + long toReviewSecurityHotspots = ((ParsedValueCount) ((ParsedFilter) categoryBucket.getAggregations().get(AGG_TO_REVIEW_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) .getValue(); - long reviewedSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get(AGG_REVIEWED_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) + long reviewedSecurityHotspots = ((ParsedValueCount) ((ParsedFilter) categoryBucket.getAggregations().get(AGG_REVIEWED_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) .getValue(); Optional<Double> percent = computePercent(toReviewSecurityHotspots, reviewedSecurityHotspots); @@ -1096,7 +1117,7 @@ public class IssueIndex { AggregationBuilders.count(AGG_COUNT).field(FIELD_ISSUE_KEY))); } - private SearchRequestBuilder prepareNonClosedVulnerabilitiesAndHotspotSearch(String projectUuid, boolean isViewOrApp) { + private static SearchSourceBuilder prepareNonClosedVulnerabilitiesAndHotspotSearch(String projectUuid, boolean isViewOrApp) { BoolQueryBuilder componentFilter = boolQuery(); if (isViewOrApp) { IndexType.IndexMainType mainType = TYPE_VIEW; @@ -1109,15 +1130,17 @@ public class IssueIndex { } else { componentFilter.filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectUuid)); } - return client.prepareSearch(TYPE_ISSUE.getMainType()) - .setQuery( + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + return sourceBuilder + .query( componentFilter .should(NON_RESOLVED_VULNERABILITIES_FILTER) .should(TO_REVIEW_HOTSPOTS_FILTER) .should(IN_REVIEW_HOTSPOTS_FILTER) .should(REVIEWED_HOTSPOTS_FILTER) .minimumShouldMatch(1)) - .setSize(0); + .size(0); } } diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java index 820b2c8033a..4231f0d42be 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java @@ -33,7 +33,6 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import javax.annotation.Nullable; import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -48,7 +47,8 @@ import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilde import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.NestedSortBuilder; import org.sonar.api.measures.Metric; @@ -119,6 +119,8 @@ import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIEL import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES_MEASURE_VALUE; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NAME; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_LANGUAGE; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_NCLOC; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ORGANIZATION_UUID; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALIFIER; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS; @@ -210,20 +212,21 @@ public class ProjectMeasuresIndex { } public SearchIdResult<String> search(ProjectMeasuresQuery query, SearchOptions searchOptions) { - SearchRequestBuilder requestBuilder = client - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setFetchSource(false) - .setFrom(searchOptions.getOffset()) - .setSize(searchOptions.getLimit()); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .fetchSource(false) + .from(searchOptions.getOffset()) + .size(searchOptions.getLimit()); AllFilters allFilters = createFilters(query); RequestFiltersComputer filtersComputer = createFiltersComputer(searchOptions, allFilters); - addFacets(requestBuilder, searchOptions, filtersComputer, query); - addSort(query, requestBuilder); - - filtersComputer.getQueryFilters().ifPresent(requestBuilder::setQuery); - filtersComputer.getPostFilters().ifPresent(requestBuilder::setPostFilter); - return new SearchIdResult<>(requestBuilder.get(), id -> id, system2.getDefaultTimeZone()); + addFacets(searchSourceBuilder, searchOptions, filtersComputer, query); + addSort(query, searchSourceBuilder); + + filtersComputer.getQueryFilters().ifPresent(searchSourceBuilder::query); + filtersComputer.getPostFilters().ifPresent(searchSourceBuilder::postFilter); + SearchResponse response = client.search(EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(searchSourceBuilder)); + return new SearchIdResult<>(response, id -> id, system2.getDefaultTimeZone()); } private static RequestFiltersComputer createFiltersComputer(SearchOptions searchOptions, AllFilters allFilters) { @@ -237,39 +240,39 @@ public class ProjectMeasuresIndex { } public ProjectMeasuresStatistics searchTelemetryStatistics() { - SearchRequestBuilder request = client - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setFetchSource(false) - .setSize(0); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .fetchSource(false) + .size(0); BoolQueryBuilder esFilter = boolQuery() .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())) .filter(termQuery(FIELD_QUALIFIER, Qualifiers.PROJECT)); - request.setQuery(esFilter); - request.addAggregation(AggregationBuilders.terms(FIELD_LANGUAGES) + searchSourceBuilder.query(esFilter); + searchSourceBuilder.aggregation(AggregationBuilders.terms(FIELD_LANGUAGES) .field(FIELD_LANGUAGES) .size(MAX_PAGE_SIZE) .minDocCount(1) .order(BucketOrder.count(false))); - request.addAggregation(AggregationBuilders.nested(FIELD_NCLOC_DISTRIBUTION, FIELD_NCLOC_DISTRIBUTION) + searchSourceBuilder.aggregation(AggregationBuilders.nested(FIELD_NCLOC_DISTRIBUTION, FIELD_NCLOC_DISTRIBUTION) .subAggregation(AggregationBuilders.terms(FIELD_NCLOC_DISTRIBUTION + "_terms") - .field(ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_LANGUAGE) + .field(FIELD_NCLOC_DISTRIBUTION_LANGUAGE) .size(MAX_PAGE_SIZE) .minDocCount(1) .order(BucketOrder.count(false)) - .subAggregation(sum(ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_NCLOC).field(ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_NCLOC)))); + .subAggregation(sum(FIELD_NCLOC_DISTRIBUTION_NCLOC).field(FIELD_NCLOC_DISTRIBUTION_NCLOC)))); - request.addAggregation(AggregationBuilders.nested(NCLOC_KEY, FIELD_MEASURES) + searchSourceBuilder.aggregation(AggregationBuilders.nested(NCLOC_KEY, FIELD_MEASURES) .subAggregation(AggregationBuilders.filter(NCLOC_KEY + "_filter", termQuery(FIELD_MEASURES_MEASURE_KEY, NCLOC_KEY)) .subAggregation(sum(NCLOC_KEY + "_filter_sum").field(FIELD_MEASURES_MEASURE_VALUE)))); ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder(); - SearchResponse response = request.get(); - statistics.setProjectCount(response.getHits().getTotalHits()); + SearchResponse response = client.search(EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(searchSourceBuilder)); + + statistics.setProjectCount(response.getHits().getTotalHits().value); statistics.setProjectCountByLanguage(termsToMap(response.getAggregations().get(FIELD_LANGUAGES))); - Function<Terms.Bucket, Long> bucketToNcloc = bucket -> Math - .round(((Sum) bucket.getAggregations().get(ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_NCLOC)).getValue()); + Function<Terms.Bucket, Long> bucketToNcloc = bucket -> Math.round(((Sum) bucket.getAggregations().get(FIELD_NCLOC_DISTRIBUTION_NCLOC)).getValue()); Map<String, Long> nclocByLanguage = Stream.of((Nested) response.getAggregations().get(FIELD_NCLOC_DISTRIBUTION)) .map(nested -> (Terms) nested.getAggregations().get(nested.getName() + "_terms")) .flatMap(terms -> terms.getBuckets().stream()) @@ -279,25 +282,25 @@ public class ProjectMeasuresIndex { return statistics.build(); } - private static void addSort(ProjectMeasuresQuery query, SearchRequestBuilder requestBuilder) { + private static void addSort(ProjectMeasuresQuery query, SearchSourceBuilder requestBuilder) { String sort = query.getSort(); if (SORT_BY_NAME.equals(sort)) { - requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), query.isAsc() ? ASC : DESC); + requestBuilder.sort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), query.isAsc() ? ASC : DESC); } else if (SORT_BY_LAST_ANALYSIS_DATE.equals(sort)) { - requestBuilder.addSort(FIELD_ANALYSED_AT, query.isAsc() ? ASC : DESC); + requestBuilder.sort(FIELD_ANALYSED_AT, query.isAsc() ? ASC : DESC); } else if (ALERT_STATUS_KEY.equals(sort)) { - requestBuilder.addSort(FIELD_QUALITY_GATE_STATUS, query.isAsc() ? ASC : DESC); - requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); + requestBuilder.sort(FIELD_QUALITY_GATE_STATUS, query.isAsc() ? ASC : DESC); + requestBuilder.sort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); } else { addMetricSort(query, requestBuilder, sort); - requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); + requestBuilder.sort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); } // last sort is by key in order to be deterministic when same value - requestBuilder.addSort(FIELD_KEY, ASC); + requestBuilder.sort(FIELD_KEY, ASC); } - private static void addMetricSort(ProjectMeasuresQuery query, SearchRequestBuilder requestBuilder, String sort) { - requestBuilder.addSort( + private static void addMetricSort(ProjectMeasuresQuery query, SearchSourceBuilder requestBuilder, String sort) { + requestBuilder.sort( new FieldSortBuilder(FIELD_MEASURES_MEASURE_VALUE) .setNestedSort( new NestedSortBuilder(FIELD_MEASURES) @@ -305,13 +308,13 @@ public class ProjectMeasuresIndex { .order(query.isAsc() ? ASC : DESC)); } - private static void addFacets(SearchRequestBuilder esRequest, SearchOptions options, RequestFiltersComputer filtersComputer, ProjectMeasuresQuery query) { + private static void addFacets(SearchSourceBuilder esRequest, SearchOptions options, RequestFiltersComputer filtersComputer, ProjectMeasuresQuery query) { TopAggregationHelper topAggregationHelper = new TopAggregationHelper(filtersComputer, new SubAggregationHelper()); options.getFacets().stream() .map(FACETS_BY_NAME::get) .filter(Objects::nonNull) .map(facet -> facet.getFacetBuilder().buildFacet(facet, query, topAggregationHelper)) - .forEach(esRequest::addAggregation); + .forEach(esRequest::aggregation); } private static AbstractAggregationBuilder<?> createRangeFacet(String metricKey, double[] thresholds) { @@ -452,14 +455,16 @@ public class ProjectMeasuresIndex { tagFacet.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(textQuery) + ".*", null)); } - SearchRequestBuilder searchQuery = client - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setQuery(authorizationTypeSupport.createQueryFilter()) - .setFetchSource(false) - .setSize(0) - .addAggregation(tagFacet); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .query(authorizationTypeSupport.createQueryFilter()) + .fetchSource(false) + .size(0) + .aggregation(tagFacet); + + SearchResponse response = client.search(EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(searchSourceBuilder)); - Terms aggregation = searchQuery.get().getAggregations().get(FIELD_TAGS); + Terms aggregation = response.getAggregations().get(FIELD_TAGS); return aggregation.getBuckets().stream() .map(Bucket::getKeyAsString) diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java index 8e0fba1e1a3..4d78374bbd7 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java @@ -24,10 +24,13 @@ import com.google.common.collect.Sets; import java.util.Map; import java.util.function.Consumer; import javax.annotation.CheckForNull; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.junit.Rule; @@ -97,19 +100,20 @@ public class IndexCreatorTest { IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake"); String id = "1"; - es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); - assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue(); + es.client().index(new IndexRequest(fakeIndexType.getIndex().getName(), fakeIndexType.getType()).id(id).source(new FakeDoc().getFields()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)); + assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName(), fakeIndexType.getType(), id)).isExists()).isTrue(); // v2 run(new FakeIndexDefinitionV2()); - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings(); - MappingMetaData mapping = mappings.get("fakes").get("fake"); + ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = mappings(); + MappingMetadata mapping = mappings.get("fakes").get("fake"); assertThat(countMappingFields(mapping)).isEqualTo(3); assertThat(field(mapping, "updatedAt").get("type")).isEqualTo("date"); assertThat(field(mapping, "newField").get("type")).isEqualTo("integer"); - assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isFalse(); + assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName(), fakeIndexType.getType(), id)).isExists()).isFalse(); } @Test @@ -207,7 +211,7 @@ public class IndexCreatorTest { private boolean isNotReadOnly(IndexMainType mainType) { String indexName = mainType.getIndex().getName(); - String readOnly = es.client().nativeClient().admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet() + String readOnly = es.client().getSettings(new GetSettingsRequest().indices(indexName)) .getSetting(indexName, "index.blocks.read_only_allow_delete"); return readOnly == null; } @@ -215,9 +219,7 @@ public class IndexCreatorTest { private void makeReadOnly(IndexMainType mainType) { Settings.Builder builder = Settings.builder(); builder.put("index.blocks.read_only_allow_delete", "true"); - es.client().nativeClient().admin().indices() - .updateSettings(new UpdateSettingsRequest().indices(mainType.getIndex().getName()).settings(builder.build())) - .actionGet(); + es.client().putSettings(new UpdateSettingsRequest().indices(mainType.getIndex().getName()).settings(builder.build())); } private void enableBlueGreenDeployment() { @@ -246,18 +248,18 @@ public class IndexCreatorTest { assertThat(es.countDocuments(FakeIndexDefinition.INDEX_TYPE)).isEqualTo(0); } - private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings() { - return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings(); + private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings() { + return es.client().getMapping(new GetMappingsRequest()).mappings(); } @CheckForNull @SuppressWarnings("unchecked") - private Map<String, Object> field(MappingMetaData mapping, String field) { + private Map<String, Object> field(MappingMetadata mapping, String field) { Map<String, Object> props = (Map<String, Object>) mapping.getSourceAsMap().get("properties"); return (Map<String, Object>) props.get(field); } - private int countMappingFields(MappingMetaData mapping) { + private int countMappingFields(MappingMetadata mapping) { return ((Map) mapping.getSourceAsMap().get("properties")).size(); } @@ -274,8 +276,8 @@ public class IndexCreatorTest { } private void verifyFakeIndexV1() { - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings(); - MappingMetaData mapping = mappings.get("fakes").get("fake"); + ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = mappings(); + MappingMetadata mapping = mappings.get("fakes").get("fake"); assertThat(mapping.type()).isEqualTo("fake"); assertThat(mapping.getSourceAsMap()).isNotEmpty(); assertThat(countMappingFields(mapping)).isEqualTo(2); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java index 835690dfdac..74d67113b28 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java @@ -23,11 +23,11 @@ import java.util.Map; import java.util.TimeZone; import org.junit.Rule; import org.junit.Test; +import org.sonar.api.impl.utils.TestSystem2; import org.sonar.api.issue.Issue; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.Severity; import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; import org.sonar.db.DbTester; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentTesting; diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java index 23164942c65..054986e98fe 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java @@ -28,9 +28,9 @@ import org.elasticsearch.search.SearchHit; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.sonar.api.impl.utils.TestSystem2; import org.sonar.api.rule.Severity; import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; import org.sonar.db.DbTester; import org.sonar.db.component.ComponentDto; import org.sonar.server.es.EsTester; diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java index b4ba5f4af5d..8563d65c54f 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.lucene.search.TotalHits; import org.assertj.core.groups.Tuple; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHit; @@ -102,15 +103,15 @@ public class IssueIndexTest { // There are 12 issues in total, with 10 issues per page, the page 2 should only contain 2 elements SearchResponse result = underTest.search(query.build(), new SearchOptions().setPage(2, 10)); assertThat(result.getHits().getHits()).hasSize(2); - assertThat(result.getHits().getTotalHits()).isEqualTo(12); + assertThat(result.getHits().getTotalHits()).isEqualTo(new TotalHits(12, TotalHits.Relation.EQUAL_TO)); result = underTest.search(IssueQuery.builder().build(), new SearchOptions().setOffset(0).setLimit(5)); assertThat(result.getHits().getHits()).hasSize(5); - assertThat(result.getHits().getTotalHits()).isEqualTo(12); + assertThat(result.getHits().getTotalHits()).isEqualTo(new TotalHits(12, TotalHits.Relation.EQUAL_TO)); result = underTest.search(IssueQuery.builder().build(), new SearchOptions().setOffset(2).setLimit(10)); assertThat(result.getHits().getHits()).hasSize(10); - assertThat(result.getHits().getTotalHits()).isEqualTo(12); + assertThat(result.getHits().getTotalHits()).isEqualTo(new TotalHits(12, TotalHits.Relation.EQUAL_TO)); } @Test diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java index b31117c203e..2011a002881 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java @@ -21,8 +21,10 @@ package org.sonar.server.permission.index; import java.util.Arrays; import java.util.List; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.es.EsClient; @@ -40,12 +42,10 @@ public class FooIndex { } public boolean hasAccessToProject(String projectUuid) { - SearchHits hits = esClient.prepareSearch(DESCRIPTOR) - .setTypes(TYPE_AUTHORIZATION.getType()) - .setQuery(QueryBuilders.boolQuery() + SearchHits hits = esClient.search(EsClient.prepareSearch(DESCRIPTOR.getName(), TYPE_AUTHORIZATION.getType()) + .source(new SearchSourceBuilder().query(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) - .filter(authorizationTypeSupport.createQueryFilter())) - .get() + .filter(authorizationTypeSupport.createQueryFilter())))) .getHits(); List<String> names = Arrays.stream(hits.getHits()) .map(h -> h.getSourceAsMap().get(FooIndexDefinition.FIELD_NAME).toString()) diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java index 696d25981c0..c2a2cbfbd47 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java @@ -22,6 +22,7 @@ package org.sonar.server.permission.index; import com.google.common.collect.ImmutableSet; import java.util.Collection; import java.util.Set; +import org.elasticsearch.action.index.IndexRequest; import org.sonar.db.DbSession; import org.sonar.db.es.EsQueueDto; import org.sonar.server.es.BaseDoc; @@ -60,11 +61,11 @@ public class FooIndexer implements ProjectIndexer, NeedAuthorizationIndexer { private void addToIndex(String projectUuid, String name) { FooDoc fooDoc = new FooDoc(projectUuid, name); - esClient.prepareIndex(TYPE_FOO) - .setId(fooDoc.getId()) - .setRouting(fooDoc.getRouting().orElse(null)) - .setSource(fooDoc.getFields()) - .get(); + esClient.index(new IndexRequest(TYPE_FOO.getMainType().getIndex().getName()) + .type(TYPE_FOO.getMainType().getType()) + .id(fooDoc.getId()) + .routing(fooDoc.getRouting().orElse(null)) + .source(fooDoc.getFields())); } private static final class FooDoc extends BaseDoc { diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java index e2e7c0e7770..b1407678103 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java @@ -19,6 +19,7 @@ */ package org.sonar.server.health; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -50,7 +51,7 @@ abstract class EsStatusCheck { Health checkEsStatus() { try { - ClusterHealthStatus esStatus = esClient.prepareClusterStats().get().getStatus(); + ClusterHealthStatus esStatus = esClient.clusterHealth(new ClusterHealthRequest()).getStatus(); if (esStatus == null) { return RED_HEALTH_UNAVAILABLE; } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java index 41a6b9c46aa..71394bc9a0f 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java @@ -293,7 +293,7 @@ public class SearchAction implements HotspotsWsAction { List<IssueDto> hotspots = toIssueDtos(dbSession, issueKeys); - Paging paging = forPageIndex(wsRequest.getPage()).withPageSize(wsRequest.getIndex()).andTotal((int) result.getHits().getTotalHits()); + Paging paging = forPageIndex(wsRequest.getPage()).withPageSize(wsRequest.getIndex()).andTotal((int) result.getHits().getTotalHits().value); return new SearchResponseData(paging, hotspots); } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/issue/ws/SearchAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/issue/ws/SearchAction.java index ead3f8bdfee..2ea5eb36b18 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/issue/ws/SearchAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/issue/ws/SearchAction.java @@ -426,7 +426,7 @@ public class SearchAction implements IssuesWsAction { SearchResponseData data = searchResponseLoader.load(preloadedData, collector, additionalFields, facets); // FIXME allow long in Paging - Paging paging = forPageIndex(options.getPage()).withPageSize(options.getLimit()).andTotal((int) result.getHits().getTotalHits()); + Paging paging = forPageIndex(options.getPage()).withPageSize(options.getLimit()).andTotal((int) result.getHits().getTotalHits().value); return searchResponseFormat.formatSearch(additionalFields, data, paging, facets); } diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java index 58668ff407f..53e83883583 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java @@ -31,6 +31,7 @@ import org.sonar.server.es.EsClient; import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.when; @@ -43,7 +44,7 @@ public class EsStatusClusterCheckTest { @Test public void check_ignores_NodeHealth_arg_and_returns_RED_with_cause_if_an_exception_occurs_checking_ES_cluster_status() { Set<NodeHealth> nodeHealths = ImmutableSet.of(newNodeHealth(NodeHealth.Status.GREEN)); - when(esClient.prepareClusterStats()).thenThrow(new RuntimeException("Faking an exception occurring while using the EsClient")); + when(esClient.clusterHealth(any())).thenThrow(new RuntimeException("Faking an exception occurring while using the EsClient")); Health health = new EsStatusClusterCheck(esClient).check(nodeHealths); @@ -54,7 +55,7 @@ public class EsStatusClusterCheckTest { @Test public void check_ignores_NodeHealth_arg_and_returns_GREEN_without_cause_if_ES_cluster_status_is_GREEN() { Set<NodeHealth> nodeHealths = ImmutableSet.of(newNodeHealth(NodeHealth.Status.YELLOW)); - when(esClient.prepareClusterStats().get().getStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esClient.clusterHealth(any()).getStatus()).thenReturn(ClusterHealthStatus.GREEN); Health health = underTest.check(nodeHealths); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java index 7786ffaab69..3d7b2641ab7 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java @@ -25,6 +25,7 @@ import org.mockito.Mockito; import org.sonar.server.es.EsClient; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -36,7 +37,7 @@ public class EsStatusNodeCheckTest { @Test public void check_ignores_NodeHealth_arg_and_returns_RED_with_cause_if_an_exception_occurs_checking_ES_cluster_status() { EsClient esClient = mock(EsClient.class); - when(esClient.prepareClusterStats()).thenThrow(new RuntimeException("Faking an exception occurring while using the EsClient")); + when(esClient.clusterHealth(any())).thenThrow(new RuntimeException("Faking an exception occurring while using the EsClient")); Health health = new EsStatusNodeCheck(esClient).check(); @@ -46,7 +47,7 @@ public class EsStatusNodeCheckTest { @Test public void check_returns_GREEN_without_cause_if_ES_cluster_status_is_GREEN() { - when(esClient.prepareClusterStats().get().getStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esClient.clusterHealth(any()).getStatus()).thenReturn(ClusterHealthStatus.GREEN); Health health = underTest.check(); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/CreateActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/CreateActionTest.java index 3d5a67462aa..cf16f7afe1a 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/CreateActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/CreateActionTest.java @@ -21,6 +21,8 @@ package org.sonar.server.user.ws; import java.util.HashSet; import java.util.Optional; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -34,6 +36,7 @@ import org.sonar.db.DbTester; import org.sonar.db.user.GroupDto; import org.sonar.db.user.UserDto; import org.sonar.server.authentication.CredentialsLocalAuthentication; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.organization.DefaultOrganizationProvider; @@ -109,13 +112,14 @@ public class CreateActionTest { .containsOnly("john", "John", "john@email.com", singletonList("jn"), true); // exists in index - assertThat(es.client().prepareSearch(UserIndexDefinition.TYPE_USER) - .setQuery(boolQuery() - .must(termQuery(FIELD_LOGIN, "john")) - .must(termQuery(FIELD_NAME, "John")) - .must(termQuery(FIELD_EMAIL, "john@email.com")) - .must(termQuery(FIELD_SCM_ACCOUNTS, "jn"))) - .get().getHits().getHits()).hasSize(1); + assertThat(es.client().search(EsClient.prepareSearch(UserIndexDefinition.TYPE_USER) + .source(new SearchSourceBuilder() + .query(boolQuery() + .must(termQuery(FIELD_LOGIN, "john")) + .must(termQuery(FIELD_NAME, "John")) + .must(termQuery(FIELD_EMAIL, "john@email.com")) + .must(termQuery(FIELD_SCM_ACCOUNTS, "jn"))))) + .getHits().getHits()).hasSize(1); // exists in db Optional<UserDto> dbUser = db.users().selectUserByLogin("john"); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java index 760f99b1adb..ae1e40c7577 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java @@ -21,6 +21,8 @@ package org.sonar.server.user.ws; import java.util.Optional; import javax.annotation.Nullable; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -43,6 +45,7 @@ import org.sonar.db.user.GroupDto; import org.sonar.db.user.SessionTokenDto; import org.sonar.db.user.UserDismissedMessageDto; import org.sonar.db.user.UserDto; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.BadRequestException; import org.sonar.server.exceptions.ForbiddenException; @@ -100,11 +103,12 @@ public class DeactivateActionTest { deactivate(user.getLogin()); verifyThatUserIsDeactivated(user.getLogin()); - assertThat(es.client().prepareSearch(UserIndexDefinition.TYPE_USER) - .setQuery(boolQuery() - .must(termQuery(FIELD_UUID, user.getUuid())) - .must(termQuery(FIELD_ACTIVE, "false"))) - .get().getHits().getHits()).hasSize(1); + assertThat(es.client().search(EsClient.prepareSearch(UserIndexDefinition.TYPE_USER) + .source(new SearchSourceBuilder() + .query(boolQuery() + .must(termQuery(FIELD_UUID, user.getUuid())) + .must(termQuery(FIELD_ACTIVE, "false"))))) + .getHits().getHits()).hasSize(1); } @Test diff --git a/sonar-application/build.gradle b/sonar-application/build.gradle index e9a4d6d8952..4b433567978 100644 --- a/sonar-application/build.gradle +++ b/sonar-application/build.gradle @@ -30,7 +30,7 @@ dependencies { // please keep list ordered compile 'org.slf4j:slf4j-api' - compile 'org.elasticsearch.client:transport' + compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client' compile project(':server:sonar-ce') compile project(':server:sonar-main') compile project(':server:sonar-process') diff --git a/sonar-application/src/main/assembly/conf/sonar.properties b/sonar-application/src/main/assembly/conf/sonar.properties index 6ce66482775..6ca7452a06e 100644 --- a/sonar-application/src/main/assembly/conf/sonar.properties +++ b/sonar-application/src/main/assembly/conf/sonar.properties @@ -271,7 +271,7 @@ # Same as previous property, but allows to not repeat all other settings like -Xmx #sonar.search.javaAdditionalOpts= -# Elasticsearch port. Default is 9001. Use 0 to get a free port. +# Elasticsearch port for incoming HTTP connections. Default is 9001. Use 0 to get a free port. # As a security precaution, should be blocked by a firewall and not exposed to the Internet. #sonar.search.port=9001 @@ -413,11 +413,3 @@ # We don't collect source code or IP addresses. And we don't share the data with anyone else. # To see an example of the data shared: login as a global administrator, call the WS api/system/info and check the Statistics field. #sonar.telemetry.enable=true - - -#-------------------------------------------------------------------------------------------------- -# DEVELOPMENT - only for developers -# The following properties MUST NOT be used in production environments. - -# Elasticsearch HTTP connector -#sonar.search.httpPort=-1 diff --git a/sonar-application/src/main/java/org/sonar/application/App.java b/sonar-application/src/main/java/org/sonar/application/App.java index 5ab3ae12142..d7f7341aaf0 100644 --- a/sonar-application/src/main/java/org/sonar/application/App.java +++ b/sonar-application/src/main/java/org/sonar/application/App.java @@ -42,7 +42,6 @@ public class App { private final JavaVersion javaVersion; private StopRequestWatcher stopRequestWatcher = null; private StopRequestWatcher hardStopRequestWatcher = null; - public App(JavaVersion javaVersion) { this.javaVersion = javaVersion; } |